Commit e6c1f9f1 authored by sfritschi's avatar sfritschi
Browse files

improved memory usage

parent 27423496
...@@ -14,7 +14,6 @@ def main(): ...@@ -14,7 +14,6 @@ def main():
print("Using %d thread(s)" % nthreads) print("Using %d thread(s)" % nthreads)
basenet = netflow.load_network_from('../netflow/network/network.h5', isGenerator=True) basenet = netflow.load_network_from('../netflow/network/network.h5', isGenerator=True)
print("Network statistics:") print("Network statistics:")
print("Throats: %d" % len(basenet.throats))
pore_throat_counts = [len(pore.throats) for pore in basenet.pores] pore_throat_counts = [len(pore.throats) for pore in basenet.pores]
print("Max. number of throats per pore: %d" % max(pore_throat_counts)) print("Max. number of throats per pore: %d" % max(pore_throat_counts))
print("Min. number of throats per pore: %d" % min(pore_throat_counts)) print("Min. number of throats per pore: %d" % min(pore_throat_counts))
......
#!/bin/bash #!/bin/bash
bsub -n 48 -W 00:30 -R fullnode python3 generate.py 48 8 8 8 bsub -n 48 -W 00:50 -R fullnode python3 generate.py 48 8 8 8
...@@ -56,7 +56,8 @@ class Network: ...@@ -56,7 +56,8 @@ class Network:
Lmax must be smaller than the smallest side length of the network ub-lb. Lmax must be smaller than the smallest side length of the network ub-lb.
""" """
throatIdx = 0 throatIdx = 0
# Globals # Globals: Lengths and radii of all throats present in base network.
# Only used for the generate_dendrogram generator
throatL = [] throatL = []
throatR = [] throatR = []
def __init__(self, lb: List[float] = [], ub: List[float] = [], \ def __init__(self, lb: List[float] = [], ub: List[float] = [], \
...@@ -113,7 +114,6 @@ class Network: ...@@ -113,7 +114,6 @@ class Network:
lt = distance(*throat_ends(throat, self.L)) lt = distance(*throat_ends(throat, self.L))
Network.throatL.append(lt) Network.throatL.append(lt)
Network.throatR.append(r) Network.throatR.append(r)
self.throats.add(throat) # TODO: Not necessary? Can potentially save memory
pore1.throats.add(Network.throatIdx) pore1.throats.add(Network.throatIdx)
pore2.throats.add(Network.throatIdx) pore2.throats.add(Network.throatIdx)
Network.throatIdx += 1 Network.throatIdx += 1
...@@ -169,9 +169,8 @@ class CellList: ...@@ -169,9 +169,8 @@ class CellList:
for i in range(self.dim)]) for i in range(self.dim)])
return cellIdx return cellIdx
def is_valid_index(self, idx: int) -> bool: # Called in parallel. Find all best neighbor pore candidates and
return 0 <= idx < self.totalCells # store their indices in poreMatchTable
def match_maker(self, pores: List[Pore], tid: int): def match_maker(self, pores: List[Pore], tid: int):
n = len(pores) n = len(pores)
for i, pore in enumerate(pores): for i, pore in enumerate(pores):
...@@ -216,25 +215,18 @@ class CellList: ...@@ -216,25 +215,18 @@ class CellList:
return candidates return candidates
# Set appropriate field in poreMatchTable to index of best candidate
def set_throat(self, poreIdx: int, matchIdx: int, idx: int): def set_throat(self, poreIdx: int, matchIdx: int, idx: int):
index = self.throatOffsets[poreIdx] + idx index = self.throatOffsets[poreIdx] + idx
# Set matchIdx # Set matchIdx
self.poreMatchTable[index] = matchIdx self.poreMatchTable[index] = matchIdx
def fetch_matches(self, poreIdx: int) -> List[int]:
lb = self.throatOffsets[poreIdx]
ub = self.throatOffsets[poreIdx + 1]
return self.poreMatchTable[lb:ub]
def fetch_throat_indices(self, poreIdx: int) -> List[int]: # Retrieve indices of best matches for given pore after they have
lb = self.throatOffsets[poreIdx] # been computed
ub = self.throatOffsets[poreIdx + 1] def fetch_matches(self, pore: Pore) -> List[int]:
return range(lb, ub) lb = self.throatOffsets[pore.index]
ub = lb + len(pore.throats)
def fetch_pos(self, poreIdx: int) -> List[float]: return self.poreMatchTable[lb:ub]
lb = self.dim * poreIdx
ub = self.dim + lb
return self.poresPos[lb:ub]
# Consider all 27 neighboring cells # Consider all 27 neighboring cells
def nbor_indices(self, pore: Pore) -> List[int]: def nbor_indices(self, pore: Pore) -> List[int]:
...@@ -655,6 +647,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \ ...@@ -655,6 +647,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
Periodic throats are marked with a periodicity label, e.g., '1 0 -1' in 3d. Periodic throats are marked with a periodicity label, e.g., '1 0 -1' in 3d.
If cutoff == float('nan'), pores are uniformly distributed. If cutoff == float('nan'), pores are uniformly distributed.
""" """
import multiprocessing as mp import multiprocessing as mp
from random import seed, random, randint from random import seed, random, randint
from itertools import product, accumulate from itertools import product, accumulate
...@@ -744,7 +737,6 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \ ...@@ -744,7 +737,6 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
n = len(pores) n = len(pores)
copies = __add_buffer_layers(pores, L, basenet.Lmax) copies = __add_buffer_layers(pores, L, basenet.Lmax)
isPeriodicCheck = 1 in targetsize
# Domain size including periodic buffer layers on all sides # Domain size including periodic buffer layers on all sides
trueDomainSize = [L[i] + 2 * basenet.Lmax for i in range(d)] trueDomainSize = [L[i] + 2 * basenet.Lmax for i in range(d)]
# Initialize cell-list; Place each pore in respective cell-set # Initialize cell-list; Place each pore in respective cell-set
...@@ -770,6 +762,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \ ...@@ -770,6 +762,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
avg_throat_diff = 0. avg_throat_diff = 0.
count = 0 count = 0
# Throats-To-Be-Realized
# Copy throats of pores to keep track of the ones already realized # Copy throats of pores to keep track of the ones already realized
ttbrs = [pore.throats.copy() for pore in pores[:n]] ttbrs = [pore.throats.copy() for pore in pores[:n]]
...@@ -804,16 +797,15 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \ ...@@ -804,16 +797,15 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
# Iterate over remaining (reduced) pores # Iterate over remaining (reduced) pores
for it, pore in enumerate(reduced_pores): for it, pore in enumerate(reduced_pores):
poreIdx = pore.index
if (not mute): if (not mute):
print(f"progress {((it+1) / n)*100.:.1f}%", end="\r", flush=True) print(f"progress {((it+1) / n)*100.:.1f}%", end="\r", flush=True)
# Set of throats to be realized # Set of throats to be realized
poreTtbr = ttbrs[poreIdx] poreTtbr = ttbrs[pore.index]
if (len(poreTtbr) == 0): continue # Nothing left to do if (len(poreTtbr) == 0): continue # Nothing left to do
matches = cellList.fetch_matches(poreIdx) matches = cellList.fetch_matches(pore)
conporesInd = set() # Set of connected pores (needed if target size contains 1) conporesInd = set() # Set of connected pores (needed if target size contains 1)
for throatIdx, match in zip(pore.throats, matches): for throatIdx, match in zip(pore.throats, matches):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment