SGCMarkus/flaredetector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

astrodatagui: CalcAllFlaresThread: split list of dictionaries

Browse Source 
previously we had a lot of overhead spawning new processes for
every single list entry.
Splitting it this way only spawns <core count> processes once,
heaving no continuous overhead.
This commit is contained in:
SGCMarkus
2024-05-15 10:59:58 +02:00
parent 724ce3e13f
commit 38aa08fcd0
1 changed files with 24 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main/astrodatagui/CalcAllFlaresThread.py
+24 -25
View File
@@ -6,24 +6,27 @@ from ..flaredetector.flaredetector import calculateFlareFitsForLightcurve
import time import time
def getFlareCount(filesDict): def getFlareCount(filesDictList):
lc = lk.read(filesDict["FilePath"]) retDictList = []
lc.flux = lc["sap_flux"] for filesDict in filesDictList:
lc_flattenend = lc.flatten() lc = lk.read(filesDict["FilePath"])
sapPeaks, sapFits = calculateFlareFitsForLightcurve(lc_flattenend) lc.flux = lc["sap_flux"]
lc.flux = lc["pdcsap_flux"] lc_flattenend = lc.flatten()
lc_flattenend = lc.flatten() sapPeaks, sapFits = calculateFlareFitsForLightcurve(lc_flattenend)
pdcsapPeaks, pdcsapFits = calculateFlareFitsForLightcurve(lc_flattenend) lc.flux = lc["pdcsap_flux"]
lc_flattenend = lc.flatten()
pdcsapPeaks, pdcsapFits = calculateFlareFitsForLightcurve(lc_flattenend)
retDict = filesDict retDict = filesDict
retDict["sapPeaks"] = sapPeaks retDict["sapPeaks"] = sapPeaks
retDict["sapPeaksCount"] = len(sapPeaks) retDict["sapPeaksCount"] = len(sapPeaks)
retDict["sapFits"] = sapFits retDict["sapFits"] = sapFits
retDict["pdcsapPeaks"] = pdcsapPeaks retDict["pdcsapPeaks"] = pdcsapPeaks
retDict["pdcsapPeaksCount"] = len(pdcsapPeaks) retDict["pdcsapPeaksCount"] = len(pdcsapPeaks)
retDict["pdcsapFits"] = pdcsapFits retDict["pdcsapFits"] = pdcsapFits
retDictList.append(retDict)
return retDict return retDictList
class CalcAllFlaresThread(QThread): class CalcAllFlaresThread(QThread):
progress = pyqtSignal(int) progress = pyqtSignal(int)
@@ -36,18 +39,14 @@ class CalcAllFlaresThread(QThread):
#@pyqtSlot() #@pyqtSlot()
def run(self): def run(self):
cpuCount = multiprocessing.cpu_count() cpuCount = multiprocessing.cpu_count()
print(len(self.allFlaresDictList)) splitList = [self.allFlaresDictList[i:i + cpuCount] for i in range(0, len(self.allFlaresDictList), cpuCount)]
start = time.time()
executor = concurrent.futures.ProcessPoolExecutor(cpuCount) executor = concurrent.futures.ProcessPoolExecutor(cpuCount)
futures = [executor.submit(getFlareCount, starDict) for starDict in self.allFlaresDictList] futures = [executor.submit(getFlareCount, starDictPartList) for starDictPartList in splitList]
concurrent.futures.wait(futures) concurrent.futures.wait(futures)
end = time.time()
print("Multiprocess time needed: ", end-start)
ret = [] ret = []
#for dic in executor.map(getFlareCount, self.allFlaresDictList):
# print(dic)
# ret.append(dic)
for future in futures: for future in futures:
ret.append(future.result()) retList = future.result()
for dic in retList:
ret.append(dic)
self.finished.emit(ret) self.finished.emit(ret)