21 """Base class for running fgcmcal on a single tract using src tables 22 or sourceTable_visit tables. 30 import lsst.pex.config
as pexConfig
31 import lsst.pipe.base
as pipeBase
33 from .fgcmBuildStars
import FgcmBuildStarsTask, FgcmBuildStarsConfig
34 from .fgcmFitCycle
import FgcmFitCycleConfig
35 from .fgcmOutputProducts
import FgcmOutputProductsTask
36 from .utilities
import makeConfigDict, translateFgcmLut, translateVisitCatalog
37 from .utilities
import computeCcdOffsets, computeApertureRadiusFromDataRef, extractReferenceMags
38 from .utilities
import makeZptSchema, makeZptCat
39 from .utilities
import makeAtmSchema, makeAtmCat
40 from .utilities
import makeStdSchema, makeStdCat
44 __all__ = [
'FgcmCalibrateTractConfigBase',
'FgcmCalibrateTractBaseTask',
'FgcmCalibrateTractRunner']
48 """Config for FgcmCalibrateTract""" 50 fgcmBuildStars = pexConfig.ConfigurableField(
51 target=FgcmBuildStarsTask,
52 doc=
"Task to load and match stars for fgcm",
54 fgcmFitCycle = pexConfig.ConfigField(
55 dtype=FgcmFitCycleConfig,
56 doc=
"Config to run a single fgcm fit cycle",
58 fgcmOutputProducts = pexConfig.ConfigurableField(
59 target=FgcmOutputProductsTask,
60 doc=
"Task to output fgcm products",
62 convergenceTolerance = pexConfig.Field(
63 doc=
"Tolerance on repeatability convergence (per band)",
67 maxFitCycles = pexConfig.Field(
68 doc=
"Maximum number of fit cycles",
72 doDebuggingPlots = pexConfig.Field(
73 doc=
"Make plots for debugging purposes?",
79 pexConfig.Config.setDefaults(self)
91 if not self.
fgcmFitCycle.useRepeatabilityForExpGrayCutsDict[band]:
92 msg =
'Must set useRepeatabilityForExpGrayCutsDict[band]=True for all bands' 93 raise pexConfig.FieldValidationError(FgcmFitCycleConfig.useRepeatabilityForExpGrayCutsDict,
98 """Subclass of TaskRunner for FgcmCalibrateTractTask 100 fgcmCalibrateTractTask.run() takes a number of arguments, one of which is 101 the butler (for persistence and mapper data), and a list of dataRefs 102 extracted from the command line. This task runs on a constrained set 103 of dataRefs, typically a single tract. 104 This class transforms the process arguments generated by the ArgumentParser 105 into the arguments expected by FgcmCalibrateTractTask.run(). 106 This runner does not use any parallelization. 112 Return a list with one element: a tuple with the butler and 115 return [(parsedCmd.butler, parsedCmd.id.refList)]
121 args: `tuple` with (butler, dataRefList) 125 exitStatus: `list` with `lsst.pipe.base.Struct` 126 exitStatus (0: success; 1: failure) 127 May also contain results if `self.doReturnResults` is `True`. 129 butler, dataRefList = args
131 task = self.TaskClass(config=self.config, log=self.log)
135 results = task.runDataRef(butler, dataRefList)
138 results = task.runDataRef(butler, dataRefList)
139 except Exception
as e:
141 task.log.fatal(
"Failed: %s" % e)
142 if not isinstance(e, pipeBase.TaskError):
143 traceback.print_exc(file=sys.stderr)
145 task.writeMetadata(butler)
147 if self.doReturnResults:
148 return [pipeBase.Struct(exitStatus=exitStatus,
151 return [pipeBase.Struct(exitStatus=exitStatus)]
155 Run the task, with no multiprocessing 159 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 164 if self.precall(parsedCmd):
166 resultList = self(targetList[0])
173 Base class to calibrate a single tract using fgcmcal 177 Instantiate an `FgcmCalibrateTractTask`. 181 butler : `lsst.daf.persistence.Butler` 184 pipeBase.CmdLineTask.__init__(self, **kwargs)
187 def _getMetadataName(self):
193 Run full FGCM calibration on a single tract, including building star list, 194 fitting multiple cycles, and making outputs. 198 butler: `lsst.daf.persistence.Butler` 199 dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` 200 Data references for the input visits. 201 These may be either per-ccd "src" or per-visit"sourceTable_visit" 206 RuntimeError: Raised if `config.fgcmBuildStars.doReferenceMatches` is 207 not True, or if fgcmLookUpTable is not available, or if 208 doSubtractLocalBackground is True and aperture radius cannot be 211 datasetType = dataRefs[0].butlerSubset.datasetType
212 self.log.info(
"Running with %d %s dataRefs" % (len(dataRefs), datasetType))
214 if not butler.datasetExists(
'fgcmLookUpTable'):
215 raise RuntimeError(
"Must run FgcmCalibrateTract with an fgcmLookUpTable")
217 if not self.config.fgcmBuildStars.doReferenceMatches:
218 raise RuntimeError(
"Must run FgcmCalibrateTract with fgcmBuildStars.doReferenceMatches")
219 if isinstance(self.config.fgcmBuildStars, FgcmBuildStarsConfig):
220 if self.config.fgcmBuildStars.checkAllCcds:
221 raise RuntimeError(
"Cannot run FgcmCalibrateTract with " 222 "fgcmBuildStars.checkAllCcds set to True")
224 self.makeSubtask(
"fgcmBuildStars", butler=butler)
225 self.makeSubtask(
"fgcmOutputProducts", butler=butler)
229 calibFluxApertureRadius =
None 230 if self.config.fgcmBuildStars.doSubtractLocalBackground:
232 field = self.config.fgcmBuildStars.instFluxField
236 raise RuntimeError(
"Could not determine aperture radius from %s. " 237 "Cannot use doSubtractLocalBackground." %
241 tract = int(dataRefs[0].dataId[
'tract'])
242 self.log.info(
"Running on tract %d" % (tract))
245 groupedDataRefs = self.fgcmBuildStars.findAndGroupDataRefs(butler, dataRefs)
246 camera = butler.get(
'camera')
247 visitCat = self.fgcmBuildStars.fgcmMakeVisitCatalog(camera, groupedDataRefs)
248 rad = calibFluxApertureRadius
249 fgcmStarObservationCat = self.fgcmBuildStars.fgcmMakeAllStarObservations(groupedDataRefs,
251 calibFluxApertureRadius=rad)
253 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = \
254 self.fgcmBuildStars.fgcmMatchStars(butler,
256 fgcmStarObservationCat)
259 lutCat = butler.get(
'fgcmLookUpTable')
261 dict(self.config.fgcmFitCycle.filterMap))
267 camera = butler.get(
'camera')
268 configDict =
makeConfigDict(self.config.fgcmFitCycle, self.log, camera,
269 self.config.fgcmFitCycle.maxIterBeforeFinalCycle,
270 True,
False, tract=tract)
272 configDict[
'doPlots'] =
False 281 noFitsDict = {
'lutIndex': lutIndexVals,
283 'expInfo': fgcmExpInfo,
284 'ccdOffsets': ccdOffsets}
286 fgcmFitCycle = fgcm.FgcmFitCycle(configDict, useFits=
False,
287 noFitsDict=noFitsDict, noOutput=
True)
292 conv = fgcmStarObservationCat[0][
'ra'].asDegrees() / float(fgcmStarObservationCat[0][
'ra'])
295 fgcmPars = fgcm.FgcmParameters.newParsWithArrays(fgcmFitCycle.fgcmConfig,
303 obsIndex = fgcmStarIndicesCat[
'obsIndex']
304 visitIndex = np.searchsorted(fgcmExpInfo[
'VISIT'],
305 fgcmStarObservationCat[
'visit'][obsIndex])
308 self.config.fgcmFitCycle.bands,
309 self.config.fgcmFitCycle.filterMap)
310 refId = fgcmRefCat[
'fgcm_id'][:]
312 fgcmStars = fgcm.FgcmStars(fgcmFitCycle.fgcmConfig)
313 fgcmStars.loadStars(fgcmPars,
314 fgcmStarObservationCat[
'visit'][obsIndex],
315 fgcmStarObservationCat[
'ccd'][obsIndex],
316 fgcmStarObservationCat[
'ra'][obsIndex] * conv,
317 fgcmStarObservationCat[
'dec'][obsIndex] * conv,
318 fgcmStarObservationCat[
'instMag'][obsIndex],
319 fgcmStarObservationCat[
'instMagErr'][obsIndex],
320 fgcmExpInfo[
'FILTERNAME'][visitIndex],
321 fgcmStarIdCat[
'fgcm_id'][:],
322 fgcmStarIdCat[
'ra'][:],
323 fgcmStarIdCat[
'dec'][:],
324 fgcmStarIdCat[
'obsArrIndex'][:],
325 fgcmStarIdCat[
'nObs'][:],
326 obsX=fgcmStarObservationCat[
'x'][obsIndex],
327 obsY=fgcmStarObservationCat[
'y'][obsIndex],
328 obsDeltaMagBkg=fgcmStarObservationCat[
'deltaMagBkg'][obsIndex],
329 psfCandidate=fgcmStarObservationCat[
'psf_candidate'][obsIndex],
339 del fgcmStarIndicesCat
342 fgcmFitCycle.setLUT(fgcmLut)
343 fgcmFitCycle.setStars(fgcmStars, fgcmPars)
348 previousReservedRawRepeatability = np.zeros(fgcmPars.nBands) + 1000.0
349 previousParInfo =
None 350 previousParams =
None 351 previousSuperStar =
None 353 while (
not converged
and cycleNumber < self.config.maxFitCycles):
355 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
359 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
366 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
367 fgcmStarObservationCat[
'instMagErr'][obsIndex])
368 fgcmFitCycle.initialCycle =
False 370 fgcmFitCycle.setPars(fgcmPars)
371 fgcmFitCycle.finishSetup()
376 previousParInfo, previousParams = fgcmFitCycle.fgcmPars.parsToArrays()
377 previousSuperStar = fgcmFitCycle.fgcmPars.parSuperStarFlat.copy()
379 self.log.info(
"Raw repeatability after cycle number %d is:" % (cycleNumber))
380 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
381 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
383 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
384 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
387 if np.alltrue((previousReservedRawRepeatability -
388 fgcmFitCycle.fgcmPars.compReservedRawRepeatability) <
389 self.config.convergenceTolerance):
390 self.log.info(
"Raw repeatability has converged after cycle number %d." % (cycleNumber))
393 fgcmFitCycle.fgcmConfig.expGrayPhotometricCut[:] = fgcmFitCycle.updatedPhotometricCut
394 fgcmFitCycle.fgcmConfig.expGrayHighCut[:] = fgcmFitCycle.updatedHighCut
395 fgcmFitCycle.fgcmConfig.precomputeSuperStarInitialCycle =
False 396 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False 397 previousReservedRawRepeatability[:] = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
398 self.log.info(
"Setting exposure gray photometricity cuts to:")
399 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
400 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
402 cut = fgcmFitCycle.updatedPhotometricCut[i] * 1000.0
403 self.log.info(
" Band %s, photometricity cut: %.2f mmag" % (band, cut))
409 self.log.warn(
"Maximum number of fit cycles exceeded (%d) without convergence." % (cycleNumber))
412 fgcmFitCycle.fgcmConfig.freezeStdAtmosphere =
False 413 fgcmFitCycle.fgcmConfig.resetParameters =
False 414 fgcmFitCycle.fgcmConfig.maxIter = 0
415 fgcmFitCycle.fgcmConfig.outputZeropoints =
True 416 fgcmFitCycle.fgcmConfig.outputStandards =
True 417 fgcmFitCycle.fgcmConfig.doPlots = self.config.doDebuggingPlots
418 fgcmFitCycle.fgcmConfig.updateCycleNumber(cycleNumber)
419 fgcmFitCycle.initialCycle =
False 421 fgcmPars = fgcm.FgcmParameters.loadParsWithArrays(fgcmFitCycle.fgcmConfig,
426 fgcmFitCycle.fgcmStars.reloadStarMagnitudes(fgcmStarObservationCat[
'instMag'][obsIndex],
427 fgcmStarObservationCat[
'instMagErr'][obsIndex])
428 fgcmFitCycle.setPars(fgcmPars)
429 fgcmFitCycle.finishSetup()
431 self.log.info(
"Running final clean-up fit cycle...")
434 self.log.info(
"Raw repeatability after clean-up cycle is:")
435 for i, band
in enumerate(fgcmFitCycle.fgcmPars.bands):
436 if not fgcmFitCycle.fgcmPars.hasExposuresInBand[i]:
438 rep = fgcmFitCycle.fgcmPars.compReservedRawRepeatability[i] * 1000.0
439 self.log.info(
" Band %s, repeatability: %.2f mmag" % (band, rep))
443 superStarChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_SSTAR_CHEB'].shape[1]
444 zptChebSize = fgcmFitCycle.fgcmZpts.zpStruct[
'FGCM_FZPT_CHEB'].shape[1]
447 zptCat =
makeZptCat(zptSchema, fgcmFitCycle.fgcmZpts.zpStruct)
450 atmCat =
makeAtmCat(atmSchema, fgcmFitCycle.fgcmZpts.atmStruct)
452 stdStruct, goodBands = fgcmFitCycle.fgcmStars.retrieveStdStarCatalog(fgcmFitCycle.fgcmPars)
454 stdCat =
makeStdCat(stdSchema, stdStruct, goodBands)
456 outStruct = self.fgcmOutputProducts.generateTractOutputProducts(butler, tract,
458 zptCat, atmCat, stdCat,
459 self.config.fgcmBuildStars,
460 self.config.fgcmFitCycle)
461 outStruct.repeatability = fgcmFitCycle.fgcmPars.compReservedRawRepeatability
def computeApertureRadiusFromDataRef(dataRef, fluxField)
def getTargetList(parsedCmd)
def makeConfigDict(config, log, camera, maxIter, resetFitParameters, outputZeropoints, tract=None)
def translateFgcmLut(lutCat, filterMap)
def computeCcdOffsets(camera, defaultOrientation)
def makeStdCat(stdSchema, stdStruct, goodBands)
def runDataRef(self, butler, dataRefs)
def extractReferenceMags(refStars, bands, filterMap)
def makeStdSchema(nBands)
def translateVisitCatalog(visitCat)
def __init__(self, butler=None, kwargs)
def makeAtmCat(atmSchema, atmStruct)
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize)
def makeZptCat(zptSchema, zpStruct)