Commit 0b425b1a authored by sfritschi's avatar sfritschi
Browse files

Finished iterative parallel approach

parent 45d1ab86
......@@ -177,15 +177,23 @@ class CellList:
for i, pore in enumerate(pores):
if (tid == 0):
print(f"progress {((i+1) / n)*100.:.1f}%", end="\r", flush=True)
if (len(pore.throats) == 0): continue # skip
candidates = self.find_candidates(pore)
# TODO: Handle case where there are no candidates left for
# some remaining throat(s)
for idx, throatIdx in enumerate(pore.throats):
lt = Network.throatL[throatIdx]
if (len(candidates) == 0):
self.set_throat(pore.index, -1, idx)
break # no possible candidates left
# Find best match among candidates
matchIdx, _ = min(candidates.items(), key=lambda il: abs(il[1] - lt))
# Remove match from candidates
del candidates[matchIdx]
# Put best match index in poreThroatTable
# Put best match index in poreMatchTable
self.set_throat(pore.index, matchIdx, idx)
# Compute dictionary of all nbor candidates of given pore
......@@ -742,6 +750,9 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
# Initialize cell-list; Place each pore in respective cell-set
cellList = CellList(pores, trueDomainSize, basenet.Lmax, basenet, n)
# Divide by two (since we are counting every throat twice)
totalThroats = len(cellList.poreMatchTable) // 2
# Evenly distribute pores (in interior) among threads
remainder = n % nthreads
loads = [n // nthreads] * nthreads
......@@ -752,18 +763,6 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
displs = [0]
displs += accumulate(loads)
workers = []
if (not mute): print("computing best matches")
# Compute best matches for each pore in parallel using available threads
for tid in range(nthreads):
poreList = pores[displs[tid] : displs[tid+1]]
worker = mp.Process(target=cellList.match_maker, args=(poreList, tid))
worker.start()
workers.append(worker)
for worker in workers:
worker.join()
# throats, connect pores
if (not mute): print("\nconnecting")
throats = []
......@@ -776,166 +775,125 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
# Copy throats of pores to keep track of the ones already realized
ttbrs = [pore.throats.copy() for pore in pores[:n]]
for poreIdx, pore in enumerate(pores[:n]):
if (not mute):
print(f"progress {((poreIdx+1) / n)*100.:.1f}%", end="\r", flush=True)
# Set of throats to be realized
poreTtbr = ttbrs[poreIdx]
if (len(poreTtbr) == 0): continue # Nothing left to do
matches = cellList.fetch_matches(poreIdx)
conporesInd = set() # Set of connected pores (needed if target size contains 1)
for throatIdx, match in zip(pore.throats, matches):
if throatIdx not in poreTtbr:
continue # Throat already realized
nbor = pores[match]
nborc = nbor # memorize potential periodic copy
# find original of buffer layer pore
if (nbor.label != LABELS[0]):
j, lbl = nbor.label.split(' ',1)
nbor = pores[int(j)]
else:
lbl = LABELS[0]
nborTtbr = ttbrs[nbor.index]
total += 1
if (len(nborTtbr) == 0 or nbor.index in conporesInd):
n_already_taken += 1
continue
conporesInd.add(nbor.index)
r = Network.throatR[throatIdx]
targetLen = Network.throatL[throatIdx]
# Compute actual throat length between pores
lt = distance(pore.pos, nborc.pos)
# connect pore to nbor
throats.append([pore, nbor, lbl, r])
# remove most similar throat of nbor
ranking = [(nborThroatIdx, abs(Network.throatL[nborThroatIdx] - lt)) \
for nborThroatIdx in nborTtbr]
nborThroatIdx, _ = min(ranking, key=lambda il: il[1])
diff = abs(targetLen - lt)
avg_throat_diff += diff
count += 1
# Successfully connected this throat
poreTtbr.remove(throatIdx)
# Remove throat from neighbor
nborTtbr.remove(nborThroatIdx)
# remove fully-connected nbor from search neighborhood
if (len(nborTtbr) == 0):
cellList.expel(nbor)
for cpore in copies[nbor]:
cellList.expel(cpore)
# remove fully-connected pore from search neighborhood
if (len(poreTtbr) == 0):
cellList.expel(pore)
for cpore in copies[pore]:
cellList.expel(cpore)
# Finally, find next best candidates for left-out pores
# in REVERSED order
if (not mute): print("\nfinding next best candidates")
if (not mute): print("Total throats: %d" % totalThroats)
# Maximum number of iterations until we give up
maxIters = 6
iterCount = 0
throatsLeft = totalThroats
throatThreshold = int(0.002 * totalThroats)
poreIdx = n
for it, pore in enumerate(reversed(pores[:n])):
poreIdx -= 1
while (iterCount < maxIters and throatsLeft > throatThreshold):
if (not mute):
print(f"progress {((it+1) / n)*100.:.1f}%", end="\r", flush=True)
if (not mute):
print("Throats left: %d (%.1f%%)" % \
(throatsLeft, throatsLeft / totalThroats * 100.))
workers = []
if (not mute): print("computing best matches")
# Compute best matches for each pore in parallel using available threads
for tid in range(nthreads):
poreList = pores[displs[tid] : displs[tid+1]]
worker = mp.Process(target=cellList.match_maker, args=(poreList, tid))
worker.start()
workers.append(worker)
poreTtbr = ttbrs[poreIdx]
if (len(poreTtbr) == 0):
continue # Nothing left to do
for worker in workers:
worker.join()
# Expensive: Find alternative (multiprocessing)
candidates = cellList.find_candidates(pore)
for poreIdx, pore in enumerate(pores[:n]):
for throatIdx in poreTtbr.copy():
if (not mute):
print(f"progress {((poreIdx+1) / n)*100.:.1f}%", end="\r", flush=True)
# Set of throats to be realized
poreTtbr = ttbrs[poreIdx]
if (len(poreTtbr) == 0): continue # Nothing left to do
r = Network.throatR[throatIdx]
targetLen = Network.throatL[throatIdx]
matches = cellList.fetch_matches(poreIdx)
if (len(candidates) == 0):
# no candidates left; give up
k += len(poreTtbr) # update num unrealized throats
break
conporesInd = set() # Set of connected pores (needed if target size contains 1)
for throatIdx, match in zip(pore.throats, matches):
nborIdx, lt = min(candidates.items(), key=lambda il: abs(il[1] - targetLen))
nbor = pores[nborIdx]
nborc = nbor # memorize potential periodic copy
# find original of buffer layer pore
if (nbor.label != LABELS[0]):
j, lbl = nbor.label.split(' ',1)
nbor = pores[int(j)]
else:
lbl = LABELS[0]
# Remove nbor from candidates as well as all periodic copies
del candidates[nborc.index]
# Remove original non-periodic nbor as well as all its
# periodic copies. Only applies if targetsize contains 1
if isPeriodicCheck:
if nbor.index in candidates:
del candidates[nbor.index]
if throatIdx not in poreTtbr:
continue # Throat already realized
if match == -1: # give up
poreTtbr.clear()
break
for cpore in copies[nbor]:
if cpore.index in candidates:
del candidates[cpore.index]
nborTtbr = ttbrs[nbor.index]
# connect pore to nbor
throats.append([pore, nbor, lbl, r])
# remove most similar throat of nbor
ranking = [(nborThroatIdx, abs(Network.throatL[nborThroatIdx] - lt)) \
for nborThroatIdx in nborTtbr]
nborThroatIdx, _ = min(ranking, key=lambda il: il[1])
diff = abs(targetLen - lt)
avg_throat_diff += diff
count += 1
nbor = pores[match]
nborc = nbor # memorize potential periodic copy
# find original of buffer layer pore
if (nbor.label != LABELS[0]):
j, lbl = nbor.label.split(' ',1)
nbor = pores[int(j)]
else:
lbl = LABELS[0]
nborTtbr = ttbrs[nbor.index]
total += 1
if (len(nborTtbr) == 0 or nbor.index in conporesInd):
n_already_taken += 1
continue
conporesInd.add(nbor.index)
r = Network.throatR[throatIdx]
targetLen = Network.throatL[throatIdx]
# Compute actual throat length between pores
lt = distance(pore.pos, nborc.pos)
# connect pore to nbor
throats.append([pore, nbor, lbl, r])
# remove most similar throat of nbor
ranking = [(nborThroatIdx, abs(Network.throatL[nborThroatIdx] - lt)) \
for nborThroatIdx in nborTtbr]
nborThroatIdx, _ = min(ranking, key=lambda il: il[1])
diff = abs(targetLen - lt)
avg_throat_diff += diff
count += 1
# Successfully connected this throat
poreTtbr.remove(throatIdx)
# Remove throat from neighbor
nborTtbr.remove(nborThroatIdx)
# remove fully-connected nbor from search neighborhood
if (len(nborTtbr) == 0):
cellList.expel(nbor)
for cpore in copies[nbor]:
cellList.expel(cpore)
# Remove throat from pore
poreTtbr.remove(throatIdx)
# Remove throat from neighbor
nborTtbr.remove(nborThroatIdx)
# remove fully-connected pore from search neighborhood
if (len(poreTtbr) == 0):
cellList.expel(pore)
# remove fully connected nbor from search neighborhood
if (len(nborTtbr) == 0):
cellList.expel(nbor)
for cpore in copies[nbor]:
for cpore in copies[pore]:
cellList.expel(cpore)
# remove fully connected pore from search neighborhood
if (len(poreTtbr) == 0):
cellList.expel(pore)
throatCount = 0
# Copy throats back to original pores and update #throats left
for i, pore in enumerate(pores[:n]):
poreTtbr = ttbrs[i]
throatCount += len(poreTtbr)
pore.throats = poreTtbr.copy()
for cpore in copies[pore]:
cellList.expel(cpore)
throatsLeft = throatCount // 2
iterCount += 1
k = throatsLeft
# Free memory associated with cellList (not needed anymore)
del cellList
# DEBUG
percent = k / (k + len(throats)) * 100.
print("\b"*24 + "{0:d} throats in total, {1:d} unrealised ({2:.1f}%)".\
if (iterCount == maxIters):
print("Maximum iteration count reached!")
percent = k / totalThroats * 100.
print("\b"*24 + "{0:d} throats in total, {1:d} unrealised ({2:.3f}%)".\
format(len(throats), k, percent))
avg_throat_diff /= count
print("Avg. throat length difference: %e" % avg_throat_diff)
......@@ -946,6 +904,7 @@ def generate_dendrogram(basenet: Network, targetsize: List[int], \
network = Network(lb=[0.0 for k in range(d)],
ub=L, Lmax=basenet.Lmax, label='from_' + basenet.label)
for pore in pores[:n]:
pore.throats.clear() # remove (potentially) remaining pores
network.add_pore(pore)
for throat in throats: network.connect_pores(pore1=throat[0],
pore2=throat[1], label=throat[2], r=throat[3])
......@@ -1215,6 +1174,7 @@ def load_network_from(filename: str, isGenerator: bool = False) -> Network:
for id, strg in zip(f['pores/label/id'],f['pores/label/strg']):
pores[id].label = strg.decode('utf-8')
# pick appropriate connector method if generation algorithm is used
connector_method = network.connect_pores
if isGenerator:
connector_method = network.connect_pores_generator
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment