Commit e6c1f9f1 authored by sfritschi's avatar sfritschi
Browse files

improved memory usage

parent 27423496
......@@ -14,7 +14,6 @@ def main():
print("Using %d thread(s)" % nthreads)
basenet = netflow.load_network_from('../netflow/network/network.h5', isGenerator=True)
print("Network statistics:")
print("Throats: %d" % len(basenet.throats))
pore_throat_counts = [len(pore.throats) for pore in basenet.pores]
print("Max. number of throats per pore: %d" % max(pore_throat_counts))
print("Min. number of throats per pore: %d" % min(pore_throat_counts))
......
#!/bin/bash
bsub -n 48 -W 00:30 -R fullnode python3 generate.py 48 8 8 8
bsub -n 48 -W 00:50 -R fullnode python3 generate.py 48 8 8 8
......@@ -56,7 +56,8 @@ class Network:
Lmax must be smaller than the smallest side length of the network ub-lb.
"""
throatIdx = 0
# Globals
# Globals: Lengths and radii of all throats present in base network.
# Only used for the generate_dendrogram generator
throatL = []
throatR = []
def __init__(self, lb: List[float] = [], ub: List[float] = [], \
......@@ -113,7 +114,6 @@ class Network:
lt = distance(*throat_ends(throat, self.L))
Network.throatL.append(lt)
Network.throatR.append(r)
self.throats.add(throat) # TODO: Not necessary? Can potentially save memory
pore1.throats.add(Network.throatIdx)
pore2.throats.add(Network.throatIdx)
Network.throatIdx += 1
......@@ -169,9 +169,8 @@ class CellList:
for i in range(self.dim)])
return cellIdx
def is_valid_index(self, idx: int) -> bool:
return 0 <= idx < self.totalCells
# Called in parallel. Find all best neighbor pore candidates and
# store their indices in poreMatchTable
def match_maker(self, pores: List[Pore], tid: int):
n = len(pores)
for i, pore in enumerate(pores):
......@@ -216,26 +215,19 @@ class CellList:
return candidates
# Set appropriate field in poreMatchTable to index of best candidate
def set_throat(self, poreIdx: int, matchIdx: int, idx: int):
index = self.throatOffsets[poreIdx] + idx
# Set matchIdx
self.poreMatchTable[index] = matchIdx
def fetch_matches(self, poreIdx: int) -> List[int]:
lb = self.throatOffsets[poreIdx]
ub = self.throatOffsets[poreIdx + 1]
# Retrieve indices of best matches for given pore after they have
# been computed
def fetch_matches(self, pore: Pore) -> List[int]:
lb = self.throatOffsets[pore.index]
ub = lb + len(pore.throats)
return self.poreMatchTable[lb:ub]
def fetch_throat_indices(self, poreIdx: int) -> List[int]:
lb = self.throatOffsets[poreIdx]
ub = self.throatOffsets[poreIdx + 1]
return range(lb, ub)
def fetch_pos(self, poreIdx: int) -> List[float]:
lb = self.dim * poreIdx
ub = self.dim + lb
return self.poresPos[lb:ub]
# Consider all 27 neighboring cells
def nbor_indices(self, pore: Pore) -> List[int]:
base = self.pore_to_index(pore)
......@@ -655,6 +647,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
Periodic throats are marked with a periodicity label, e.g., '1 0 -1' in 3d.
If cutoff == float('nan'), pores are uniformly distributed.
"""
import multiprocessing as mp
from random import seed, random, randint
from itertools import product, accumulate
......@@ -744,7 +737,6 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
n = len(pores)
copies = __add_buffer_layers(pores, L, basenet.Lmax)
isPeriodicCheck = 1 in targetsize
# Domain size including periodic buffer layers on all sides
trueDomainSize = [L[i] + 2 * basenet.Lmax for i in range(d)]
# Initialize cell-list; Place each pore in respective cell-set
......@@ -770,6 +762,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
avg_throat_diff = 0.
count = 0
# Throats-To-Be-Realized
# Copy throats of pores to keep track of the ones already realized
ttbrs = [pore.throats.copy() for pore in pores[:n]]
......@@ -804,16 +797,15 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
# Iterate over remaining (reduced) pores
for it, pore in enumerate(reduced_pores):
poreIdx = pore.index
if (not mute):
print(f"progress {((it+1) / n)*100.:.1f}%", end="\r", flush=True)
# Set of throats to be realized
poreTtbr = ttbrs[poreIdx]
poreTtbr = ttbrs[pore.index]
if (len(poreTtbr) == 0): continue # Nothing left to do
matches = cellList.fetch_matches(poreIdx)
matches = cellList.fetch_matches(pore)
conporesInd = set() # Set of connected pores (needed if target size contains 1)
for throatIdx, match in zip(pore.throats, matches):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment