SGCMarkus/flaredetector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

generate_plots: use multiprocessing

Browse Source
This commit is contained in:
SGCMarkus
2025-05-06 11:48:01 +02:00
parent 43eb79bcc6
commit 7d574febcf
1 changed files with 515 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generate_plots.py
+109 -83
View File
@@ -1,3 +1,4 @@
from math import comb
import numpy as np
import pandas as pd
import itertools
@@ -9,6 +10,10 @@ from datetime import datetime
from errno import EEXIST
from os import makedirs, path
import shutil
import multiprocessing
import concurrent.futures
from concurrent.futures import wait, ALL_COMPLETED
from functools import partial
def normalizePhase(phase, phaseMin = None, phaseMax = None):
if(phaseMin is None):
@@ -27,41 +32,6 @@ def mkdir_p(mypath):
pass
else: raise
fileName = "datav5.1.cff"
data = pd.read_pickle(fileName)
binList = [10, 20, 30]
spType = ["M", "K", "G", "F"]
useKepler = True
useK2 = True
useTESS = True
showSourceFilter = np.full(len(data), False)
if(useKepler):
showKepler = data["Source"] == "Kepler"
showSourceFilter |= showKepler
if(useK2):
showK2 = data["Source"] == "K2"
showSourceFilter |= showK2
if(useTESS):
showTESS = data["Source"] == "TESS"
showSourceFilter |= showTESS
current = datetime.now()
date = f"{current.year}-{current.month}-{current.day}"
time = f"{current.hour}-{current.minute}-{current.second}"
folderPath = f"../{date}-poly-only/"
#folderPath = f"G:/Meine Ablage/Masterthesis/{date}/"
mkdir_p(folderPath)
# remove any data that has no period
#data = data[(data["FitType"] == "sine") | (data["FitType"] == "poly")]
data = data[((data["FitType"] == "poly")) & (data["isValidFold"])]
validStarPeriodMap = []
starList = set(list(data["StarName"]))
def allValuesWithin3Std(values: list):
if(not values or len(values) == 1):
return True
@@ -71,19 +41,6 @@ def allValuesWithin3Std(values: list):
def getMeanPeriod(values: list):
return np.mean(values)
for starName in starList:
periods = list(data[data["StarName"] == starName]["pdcsapPeriod"])
validStarPeriodMap.append({"StarName": starName,
"MeanPeriod": getMeanPeriod(periods),
"PeriodWithinStd": allValuesWithin3Std(periods)})
validStarPeriodMap = pd.DataFrame(validStarPeriodMap)
periodsCutList = [0.5, 1, 1.5, 2, 5, 10, 15, 20]
data = pd.merge(data, validStarPeriodMap, on="StarName")
starDB: StarDB = StarDB.getInstance("stars.db")
def plotBinsHistogram(PDCSAPdataList, PDCSAPlabelList, PDCSAPcolorList, bins, title, filename, foldedFits):
figHisto, ((axHisto)) = plt.subplots(nrows=1, ncols=1)
y, binEdges, _ = axHisto.hist(PDCSAPdataList, bins,
@@ -182,9 +139,8 @@ def plotFlarePeaks(plotdata, filters, labels, colors, maxY, title, filename):
plt.savefig(filename)
plt.close()
def setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak,
PDCSAPdataList=None, dataFilter=None,
PeriodModulation=False):
def setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak, pdcsapbinningData,
pdcsapbinningDataColumn, PDCSAPdataList=None, dataFilter=None):
xData = pd.DataFrame()
yData = pd.DataFrame()
for aX, aY in zip(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak):
@@ -195,15 +151,9 @@ def setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak,
yData = np.asarray(yData.values)[:,0]
if(PDCSAPdataList is not None):
if(dataFilter is not None):
if(PeriodModulation):
PDCSAPdataList.append(pdcsapbinningDataSpotModDiffPeriod[dataFilter]["PDCSAPNormPhasePeriod"])
PDCSAPdataList.append(pdcsapbinningData[dataFilter][pdcsapbinningDataColumn])
else:
PDCSAPdataList.append(pdcsapbinningData[dataFilter]["PDCSAPNormPhase"])
else:
if(PeriodModulation):
PDCSAPdataList.append(pdcsapbinningDataSpotModDiffPeriod[:]["PDCSAPNormPhasePeriod"])
else:
PDCSAPdataList.append(pdcsapbinningData[:]["PDCSAPNormPhase"])
PDCSAPdataList.append(pdcsapbinningData[:][pdcsapbinningDataColumn])
return xData, yData, PDCSAPdataList
@@ -226,8 +176,7 @@ def generatePlots(PDCSAPdataList, PDCSAPlabelList, PDCSAPcolorList, histogramTit
flarePlotFilename = flarePlotFilenameArg.replace("@maxY", str(maxY))
plotFlarePeaks(plotdata, filters, flarePlotLabels, flarePlotColors, maxY, flarePlotTitle, flarePlotFilename)
# All stars
for starName in starDB.getAllStars():
def plotStar(data, showSourceFilter, folderPath, starName):
finalData = pd.DataFrame()
starNameR = starName.replace('*', '_star_')
nameFilter = data["StarName"] == starName
@@ -292,11 +241,11 @@ for starName in starDB.getAllStars():
PDCSAPdataList2dhistPhase.append(pdcsapbinningData[:]["PDCSAPNormPhase"])
PDCSAPdataList2dhistPeak.append(pdcsapbinningData[:]["Peak"])
xData, yData, PDCSAPdataList = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak, PDCSAPdataList)
xData, yData, PDCSAPdataList = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak,
pdcsapbinningData, "PDCSAPNormPhase",
PDCSAPdataList)
PDCSAPlabelList.append(f"{starName}")
PDCSAPcolorList.append(color)
generatePlots(PDCSAPdataList, PDCSAPlabelList, PDCSAPcolorList, f"Flare count in phase of {starName} with @bins bins", f"{locFolder}/{starNameR}-Flarecount-@bins_Bins.png",
xData, yData, f"Flare peak per phase histogram of {starName} with @bins bins", f"{locFolder}/{starNameR}-Flarepeaks-@bins_Bins_maxY-@maxY.png",
pdcsapbinningData, None, f"{starName}", color, f"Flare peaks per phase of {starName}", f"{locFolder}/{starNameR}-Flarepeaks_maxY-@maxY.png",
@@ -323,7 +272,9 @@ for starName in starDB.getAllStars():
PDCSAPdataList2dhistPhase.append(pdcsapbinningDataSpotModDiffPeriod[:]["PDCSAPNormPhasePeriod"])
PDCSAPdataList2dhistPeak.append(pdcsapbinningDataSpotModDiffPeriod[:]["PeakPeriod"])
xData, yData, PDCSAPdataListPeriod = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak, PDCSAPdataListPeriod, PeriodModulation=True)
xData, yData, PDCSAPdataListPeriod = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak,
pdcsapbinningDataSpotModDiffPeriod, "PDCSAPNormPhasePeriod",
PDCSAPdataListPeriod)
PDCSAPlabelList.append(f"{starName}")
PDCSAPcolorList.append(color)
@@ -333,9 +284,7 @@ for starName in starDB.getAllStars():
pdcsapbinningDataSpotModDiffPeriod, None, f"{starName}", color, f"Flare peaks per phase of {starName}", f"{locFolder}/{starNameR}-Flarepeaks_maxY-@maxY_Period.png",
foldedPeriodFits)
for comboLength in range(1, len(spType) + 1):
for combo in itertools.combinations(spType, comboLength):
def plotCombo(data, showSourceFilter, folderPath, combo):
for maxFlarePeak in [1.01, 1.05, 1.1, 1.25, 1.5]:
# max Flare Peak cut
finalDataMaxFlarePeak = pd.DataFrame()
@@ -438,10 +387,11 @@ for comboLength in range(1, len(spType) + 1):
PDCSAPcolorListU.append("greenyellow")
plotFiltersU.append(Ffilter)
xData, yData, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseU, PDCSAPdataList2dhistPeakU)
generatePlots(PDCSAPdataListU, PDCSAPlabelListU, PDCSAPcolorListU, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolder}/{''.join(combo)}_maxFlarePeak_{maxFlarePeak}-Flarecount-@bins_Bins.png",
xData, yData, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolder}/{''.join(combo)}_maxFlarePeak_{maxFlarePeak}-Flarepeaks-@bins_Bins_maxY-@maxY.png",
pdcsapbinningDataU, plotFiltersU, PDCSAPlabelListU, PDCSAPcolorListU, f"Flare peaks per phase of {', '.join(combo)} type stars ({numStars} stars)", f"{locFolder}/{''.join(combo)}_maxFlarePeak_{maxFlarePeak}-Flarepeaks_maxY-@maxY.png")
xData, yData, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseU, PDCSAPdataList2dhistPeakU,
pdcsapbinningDataU, "PDCSAPNormPhase")
generatePlots(PDCSAPdataListU, PDCSAPlabelListU, PDCSAPcolorListU, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolderU}/{''.join(combo)}_maxFlarePeak_{maxFlarePeak}-Flarecount-@bins_Bins.png",
xData, yData, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolderU}/{''.join(combo)}_maxFlarePeak_{maxFlarePeak}-Flarepeaks-@bins_Bins_maxY-@maxY.png",
pdcsapbinningDataU, plotFiltersU, PDCSAPlabelListU, PDCSAPcolorListU, f"Flare peaks per phase of {', '.join(combo)} type stars ({numStars} stars)", f"{locFolderU}/{''.join(combo)}_maxFlarePeak_{maxFlarePeak}-Flarepeaks_maxY-@maxY.png")
if(len(pdcsapbinningDataO) > 0):
@@ -485,10 +435,11 @@ for comboLength in range(1, len(spType) + 1):
PDCSAPcolorListO.append("greenyellow")
plotFiltersO.append(Ffilter)
xData, yData, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseO, PDCSAPdataList2dhistPeakO)
generatePlots(PDCSAPdataListO, PDCSAPlabelListO, PDCSAPcolorListO, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolder}/{''.join(combo)}_minFlarePeak_{maxFlarePeak}-Flarecount-@bins_Bins.png",
xData, yData, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolder}/{''.join(combo)}_minFlarePeak_{maxFlarePeak}-Flarepeaks-@bins_Bins_maxY-@maxY.png",
pdcsapbinningDataO, plotFiltersO, PDCSAPlabelListO, PDCSAPcolorListO, f"Flare peaks per phase of {', '.join(combo)} type stars ({numStars} stars)", f"{locFolder}/{''.join(combo)}_minFlarePeak_{maxFlarePeak}-Flarepeaks_maxY-@maxY.png")
xData, yData, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseO, PDCSAPdataList2dhistPeakO,
pdcsapbinningDataO, "PDCSAPNormPhase")
generatePlots(PDCSAPdataListO, PDCSAPlabelListO, PDCSAPcolorListO, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolderO}/{''.join(combo)}_minFlarePeak_{maxFlarePeak}-Flarecount-@bins_Bins.png",
xData, yData, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolderO}/{''.join(combo)}_minFlarePeak_{maxFlarePeak}-Flarepeaks-@bins_Bins_maxY-@maxY.png",
pdcsapbinningDataO, plotFiltersO, PDCSAPlabelListO, PDCSAPcolorListO, f"Flare peaks per phase of {', '.join(combo)} type stars ({numStars} stars)", f"{locFolderO}/{''.join(combo)}_minFlarePeak_{maxFlarePeak}-Flarepeaks_maxY-@maxY.png")
# all flare peaks
finalDataAllFlarePeaks = pd.DataFrame()
if("M" in combo):
@@ -575,14 +526,15 @@ for comboLength in range(1, len(spType) + 1):
PDCSAPcolorList.append("greenyellow")
plotFilters.append(Ffilter)
xData, yData, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak)
xData, yData, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak,
pdcsapbinningData, "PDCSAPNormPhase",)
plotdata = pdcsapbinningData
filters = plotFilters
generatePlots(PDCSAPdataList, PDCSAPlabelList, PDCSAPcolorList, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolder}/{''.join(combo)}-Flarecount-@bins_Bins.png",
xData, yData, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins ({numStars} stars)", f"{locFolder}/{''.join(combo)}-Flarepeaks-@bins_Bins_maxY-@maxY.png",
plotdata, filters, PDCSAPlabelList, PDCSAPcolorList, f"Flare peaks per phase of {', '.join(combo)} type stars ({numStars} stars)", f"{locFolder}/{''.join(combo)}-Flarepeaks_maxY-@maxY.png")
if(comboLength == 1):
if(len(combo) == 1):
mainSpType = combo[0]
spTypes = [f"{mainSpType}0", f"{mainSpType}1", f"{mainSpType}2", f"{mainSpType}3", f"{mainSpType}4", f"{mainSpType}5", f"{mainSpType}6", f"{mainSpType}7", f"{mainSpType}8", f"{mainSpType}9"]
match mainSpType:
@@ -642,7 +594,9 @@ for comboLength in range(1, len(spType) + 1):
PDCSAPdataList2dhistPeak.append(pdcsapbinningData[SpTypefilter]["Peak"])
PDCSAPlabelList.append(f"{spTyp} Stars")
PDCSAPcolorList.append(color)
xData, yData, PDCSAPdataList = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak, PDCSAPdataList, SpTypefilter)
xData, yData, PDCSAPdataList = setupxyDataAndDataList(PDCSAPdataList2dhistPhase, PDCSAPdataList2dhistPeak,
pdcsapbinningData, "PDCSAPNormPhase",
PDCSAPdataList, SpTypefilter)
plotdata = pdcsapbinningData
filters = SpTypefilter
@@ -692,13 +646,13 @@ for comboLength in range(1, len(spType) + 1):
for td, peak, pv in zip(pdcsapVals["Phase"], pdcsapVals["Peak"], row["pdcsapPeaks"]):
normPhase = normalizePhase(td.value, np.abs(PDCSAPminOrigPhase), np.abs(PDCSAPmaxOrigPhase))
if(row["MeanPeriod"] <= periodCut):
csvFileU.write(f"{row['StarName']},{row['SpType']},{row['Source']},{row['FilePath']},{pv['FlarePeakTime']},{pv['FlarePeak']},{row['pdcsapPeriod']},{row["MeanPeriod"]},{normPhase},{peak['FlarePeak']}")
csvFileU.write(f"{row['StarName']},{row['SpType']},{row['Source']},{row['FilePath']},{pv['FlarePeakTime']},{pv['FlarePeak']},{row['pdcsapPeriod']},{row['MeanPeriod']},{normPhase},{peak['FlarePeak']}")
csvFileU.write("\n")
pdcsapbinningDataU.append({"SpType": f'{row["SpType"][0]}',
"PDCSAPNormPhase": normPhase,
"Peak": peak["FlarePeak"]})
else:
csvFileO.write(f"{row['StarName']},{row['SpType']},{row['Source']},{row['FilePath']},{pv['FlarePeakTime']},{pv['FlarePeak']},{row['pdcsapPeriod']},{row["MeanPeriod"]},{normPhase},{peak['FlarePeak']}")
csvFileO.write(f"{row['StarName']},{row['SpType']},{row['Source']},{row['FilePath']},{pv['FlarePeakTime']},{pv['FlarePeak']},{row['pdcsapPeriod']},{row['MeanPeriod']},{normPhase},{peak['FlarePeak']}")
csvFileO.write("\n")
pdcsapbinningDataO.append({"SpType": f'{row["SpType"][0]}',
"PDCSAPNormPhase": normPhase,
@@ -782,16 +736,88 @@ for comboLength in range(1, len(spType) + 1):
plotFiltersO.append(FfilterO)
if(len(PDCSAPdataList2dhistPhaseU) > 0 and len(PDCSAPdataList2dhistPeakU) > 0):
xDataU, yDataU, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseU, PDCSAPdataList2dhistPeakU)
xDataU, yDataU, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseU, PDCSAPdataList2dhistPeakU,
pdcsapbinningDataU, "PDCSAPNormPhase",)
if(len(PDCSAPdataListU) > 0 and len(xDataU) > 0 and len(yDataU) > 0):
generatePlots(PDCSAPdataListU, PDCSAPlabelList, PDCSAPcolorList, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins (Rot. Period under {periodCut} days)", f"{locFolder}/{''.join(combo)}-Flarecount-Period_u_{periodCut}-@bins_Bins.png",
xDataU, yDataU, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins (Rot. Period under {periodCut} days)", f"{locFolder}/{''.join(combo)}-Flarepeaks-Period_u_{periodCut}-@bins_Bins_maxY-@maxY.png",
pdcsapbinningDataU, plotFiltersU, PDCSAPlabelList, PDCSAPcolorList, f"Flare peaks per phase of {', '.join(combo)} type stars (Rot. Period under {periodCut} days)", f"{locFolder}/{''.join(combo)}-Flarepeaks-Period_u_{periodCut}-maxY_@maxY.png")
if(len(PDCSAPdataList2dhistPhaseO) > 0 and len(PDCSAPdataList2dhistPeakO) > 0):
xDataO, yDataO, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseO, PDCSAPdataList2dhistPeakO)
xDataO, yDataO, _ = setupxyDataAndDataList(PDCSAPdataList2dhistPhaseO, PDCSAPdataList2dhistPeakO,
pdcsapbinningDataO, "PDCSAPNormPhase",)
if(len(PDCSAPdataListO) > 0 and len(xDataO) > 0 and len(yDataO) > 0):
generatePlots(PDCSAPdataListO, PDCSAPlabelList, PDCSAPcolorList, f"Flare count per phase of {', '.join(combo)} type stars with @bins bins (Rot. Period over {periodCut} days)", f"{locFolder}/{''.join(combo)}-Flarecount-Period_o_{periodCut}-@bins_Bins.png",
xDataO, yDataO, f"Flare peak per phase histogram of {', '.join(combo)} type stars with @bins bins (Rot. Period over {periodCut} days)", f"{locFolder}/{''.join(combo)}-Flarepeaks-Period_o_{periodCut}-@bins_Bins_maxY-@maxY.png",
pdcsapbinningDataO, plotFiltersO, PDCSAPlabelList, PDCSAPcolorList, f"Flare peaks per phase of {', '.join(combo)} type stars (Rot. Period over {periodCut} days)", f"{locFolder}/{''.join(combo)}-Flarepeaks-Period_o_{periodCut}-maxY_@maxY.png")
binList = [10, 20, 30]
spType = ["M", "K", "G", "F"]
if __name__ == "__main__":
fileName = "datav5.1.cff"
fullData = pd.read_pickle(fileName)
starDB: StarDB = StarDB.getInstance("stars.db")
useKepler = True
useK2 = True
useTESS = True
current = datetime.now()
date = f"{current.year}-{current.month}-{current.day}"
time = f"{current.hour}-{current.minute}-{current.second}"
cpuCount = multiprocessing.cpu_count()
executor = concurrent.futures.ProcessPoolExecutor(cpuCount)
#executor = concurrent.futures.ThreadPoolExecutor(cpuCount)
foldedFitTypes = ["sine", "poly"]
for foldedFitTypesLength in range(1, len(foldedFitTypes)+1):
for foldedFitTypeCombo in itertools.combinations(foldedFitTypes, foldedFitTypesLength):
folderPath = f"../{date}-{'-'.join(foldedFitTypeCombo)}/"
mkdir_p(folderPath)
foldedFitTypeComboFilter = np.full(len(fullData), False)
if("sine" in foldedFitTypeCombo):
foldedFitTypeComboFilter |= fullData["FitType"] == "sine"
if("poly" in foldedFitTypeCombo):
foldedFitTypeComboFilter |= fullData["FitType"] == "poly"
data = fullData[(foldedFitTypeComboFilter) & (fullData["isValidFold"])]
showSourceFilter = np.full(len(data), False)
if(useKepler):
showKepler = data["Source"] == "Kepler"
showSourceFilter |= showKepler
if(useK2):
showK2 = data["Source"] == "K2"
showSourceFilter |= showK2
if(useTESS):
showTESS = data["Source"] == "TESS"
showSourceFilter |= showTESS
validStarPeriodMap = []
starList = set(list(data["StarName"]))
for starName in starList:
periods = list(data[data["StarName"] == starName]["pdcsapPeriod"])
validStarPeriodMap.append({"StarName": starName,
"MeanPeriod": getMeanPeriod(periods),
"PeriodWithinStd": allValuesWithin3Std(periods)})
validStarPeriodMap = pd.DataFrame(validStarPeriodMap)
periodsCutList = [0.5, 1, 1.5, 2, 5, 10, 15, 20]
data = pd.merge(data, validStarPeriodMap, on="StarName")
starPlotFunc = partial(plotStar, data, showSourceFilter, folderPath)
list(executor.map(starPlotFunc, starDB.getAllStars())) # wrap in list, to force evaluation
combos = []
for comboLength in range(1, len(spType) + 1):
for combo in itertools.combinations(spType, comboLength):
combos.append(combo)
plotComboFunc = partial(plotCombo, data, showSourceFilter, folderPath)
list(executor.map(plotComboFunc, combos))