astrodatagui: use pandas instead of lists for faster processing

main/astrodatagui/AstrodataGUI.py

@@ -1,10 +1,10 @@
-from msilib import sequence
 from PyQt5 import QtWidgets, uic, Qt
 from PyQt5 import QtCore
 import os
 
 from astroquery.simbad import Simbad
 import numpy as np
+import pandas as pd
 
 from .db.StarsDB import StarDB
 from .ui.NewStarDialog import NewStarDialog
@@ -359,7 +359,15 @@ class AstrodataGUI(QtWidgets.QMainWindow):
 
     def btCountAllFlaresClicked(self):
         if(self.calcAllFlaresThread is None):
-            allStarsDictList = []
+            allStarsDictList = pd.DataFrame(columns=["StarName",
+                                                     "SpType",
+                                                     "RotVel",
+                                                     "RotVelUnit",
+                                                     "Distance",
+                                                     "DistanceUnit",
+                                                     "Source",
+                                                     "Sequence",
+                                                     "FilePath"])
 
             for starName in self.starDB.getAllStars():
                 sequences = self.starDB.getStarSequences(starName)
@@ -368,16 +376,10 @@ class AstrodataGUI(QtWidgets.QMainWindow):
                     source = sourceSeq["Source"]
                     seq = sourceSeq["Sequence"]
                     filePath = self.starDB.getFilePath(starName, source, seq)
-                    allStarsDictList.append({"StarName": starName,
-                                             "SpType": infos["SpType"],
-                                             "RotVel": infos["RotVel"],
-                                             "RotVelUnit": infos["RotVelUnit"],
-                                             "Distance": infos["Distance"],
-                                             "DistanceUnit": infos["DistanceUnit"],
-                                             "Source": source,
-                                             "Sequence": seq,
-                                             "FilePath": filePath})
+                    allStarsDictList.loc[len(allStarsDictList.index)] = \
+                            [starName, infos["SpType"], infos["RotVel"], infos["RotVelUnit"],
+                             infos["Distance"], infos["DistanceUnit"], source, seq, filePath]
 
             self.calcAllFlaresThread = CalcAllFlaresThread(allStarsDictList)
             self.calcAllFlaresThread.finished.connect(self.btCountAllFlaresClickedDone)
-            self.calcAllFlaresThread.start()
+            self.calcAllFlaresThread.start()

main/astrodatagui/CalcAllFlaresThread.py

@@ -2,31 +2,41 @@ from PyQt5.QtCore import pyqtSignal, QThread
 import multiprocessing
 import concurrent.futures
 import lightkurve as lk
+import pandas as pd
 from ..flaredetector.flaredetector import calculateFlareFitsForLightcurve
 
-import time
-
 def getFlareCount(filesDictList):
-    retDictList = []
-    for filesDict in filesDictList:
-        lc = lk.read(filesDict["FilePath"])
+    sapPeaksL = []
+    sapPeaksCountL = []
+    sapFitsL = []
+    pdcsapPeaksL = []
+    pdcsapPeaksCountL = []
+    pdcsapFitsL = []
+    for ind in filesDictList.index:
+        lc = lk.read(filesDictList["FilePath"][ind])
         lc.flux = lc["sap_flux"]
-        lc_flattenend = lc.flatten()
-        sapPeaks, sapFits = calculateFlareFitsForLightcurve(lc_flattenend)
+        lc.flux_err = lc["sap_flux_err"]
+        sapPeaks, sapFits = calculateFlareFitsForLightcurve(lc.flatten())
         lc.flux = lc["pdcsap_flux"]
-        lc_flattenend = lc.flatten()
-        pdcsapPeaks, pdcsapFits = calculateFlareFitsForLightcurve(lc_flattenend)
+        lc.flux_err = lc["pdcsap_flux_err"]
+        pdcsapPeaks, pdcsapFits = calculateFlareFitsForLightcurve(lc.flatten())
 
-        retDict = filesDict
-        retDict["sapPeaks"] = sapPeaks
-        retDict["sapPeaksCount"] = len(sapPeaks)
-        retDict["sapFits"] = sapFits
-        retDict["pdcsapPeaks"] = pdcsapPeaks
-        retDict["pdcsapPeaksCount"] = len(pdcsapPeaks)
-        retDict["pdcsapFits"] = pdcsapFits
-        retDictList.append(retDict)
+        sapPeaksL.append(sapPeaks)
+        sapPeaksCountL.append(len(sapPeaks))
+        sapFitsL.append(sapFits)
+        pdcsapPeaksL.append(pdcsapPeaks)
+        pdcsapPeaksCountL.append(len(pdcsapPeaks))
+        pdcsapFitsL.append(pdcsapFits)
+        del lc
 
-    return retDictList
+    filesDictList["sapPeaks"] = sapPeaksL
+    filesDictList["sapPeaksCount"] = sapPeaksCountL
+    filesDictList["sapFits"] = sapFitsL
+    filesDictList["pdcsapPeaks"] = pdcsapPeaksL
+    filesDictList["pdcsapPeaksCount"] = pdcsapPeaksCountL
+    filesDictList["pdcsapFits"] = pdcsapFitsL
+
+    return filesDictList
 
 class CalcAllFlaresThread(QThread):
     progress = pyqtSignal(int)
@@ -43,10 +53,9 @@ class CalcAllFlaresThread(QThread):
         executor = concurrent.futures.ProcessPoolExecutor(cpuCount)
         futures = [executor.submit(getFlareCount, starDictPartList) for starDictPartList in splitList]
         concurrent.futures.wait(futures)
-        ret = []
+        ret = pd.DataFrame()
         for future in futures:
-            retList = future.result()
-            for dic in retList:
-                ret.append(dic)
+            retFrame = future.result()
+            ret = pd.concat([ret, retFrame], ingore_index=True)
 
         self.finished.emit(ret)