23 """Build star observations for input to FGCM using sourceTable_visit. 25 This task finds all the visits and sourceTable_visits in a repository (or a 26 subset based on command line parameters) and extracts all the potential 27 calibration stars for input into fgcm. This task additionally uses fgcm to 28 match star observations into unique stars, and performs as much cleaning of the 29 input catalog as possible. 37 import lsst.pex.config
as pexConfig
38 import lsst.pipe.base
as pipeBase
39 import lsst.afw.table
as afwTable
41 from .fgcmBuildStarsBase
import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
42 from .utilities
import computeApproxPixelAreaFields
44 __all__ = [
'FgcmBuildStarsTableConfig',
'FgcmBuildStarsTableTask']
48 """Config for FgcmBuildStarsTableTask""" 50 referenceCCD = pexConfig.Field(
51 doc=
"Reference CCD for checking PSF and background",
71 fluxFlagName = self.
instFluxField[0: -len(
'instFlux')] +
'flag' 73 sourceSelector.flags.bad = [
'PixelFlags_edge',
74 'PixelFlags_interpolatedCenter',
75 'PixelFlags_saturatedCenter',
76 'PixelFlags_crCenter',
78 'PixelFlags_interpolated',
79 'PixelFlags_saturated',
85 sourceSelector.flags.bad.append(localBackgroundFlagName)
88 sourceSelector.signalToNoise.errField = self.
instFluxField +
'Err' 90 sourceSelector.isolated.parentName =
'parentSourceId' 91 sourceSelector.isolated.nChildName =
'Deblend_nChild' 93 sourceSelector.unresolved.name =
'extendedness' 98 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 100 ConfigClass = FgcmBuildStarsTableConfig
101 RunnerClass = FgcmBuildStarsRunner
102 _DefaultName =
"fgcmBuildStarsTable" 105 def _makeArgumentParser(cls):
106 """Create an argument parser""" 108 parser.add_id_argument(
"--id",
"sourceTable_visit", help=
"Data ID, e.g. --id visit=6789")
113 self.log.info(
"Grouping dataRefs by %s" % (self.config.visitDataRefName))
115 camera = butler.get(
'camera')
118 for detector
in camera:
119 ccdIds.append(detector.getId())
123 ccdIds.insert(0, self.config.referenceCCD)
130 groupedDataRefs = collections.defaultdict(list)
131 for dataRef
in dataRefs:
132 visit = dataRef.dataId[self.config.visitDataRefName]
138 calexpRef = butler.dataRef(
'calexp', dataId={self.config.visitDataRefName: visit,
139 self.config.ccdDataRefName: ccdId})
145 if not calexpRef.datasetExists():
150 groupedDataRefs[visit].append(calexpRef)
154 groupedDataRefs[visit].append(dataRef)
156 return groupedDataRefs
159 calibFluxApertureRadius=None,
160 visitCatDataRef=None,
163 startTime = time.time()
169 if (visitCatDataRef
is not None and starObsDataRef
is None or 170 visitCatDataRef
is None and starObsDataRef
is not None):
171 self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so " 172 "no checkpoint files will be persisted.")
174 if self.config.doSubtractLocalBackground
and calibFluxApertureRadius
is None:
175 raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
179 dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
180 sourceSchema = dataRef.get(
'src_schema', immediate=
True).schema
182 outputSchema = sourceMapper.getOutputSchema()
185 camera = dataRef.get(
'camera')
187 for ccdIndex, detector
in enumerate(camera):
188 ccdMapping[detector.getId()] = ccdIndex
192 if inStarObsCat
is not None:
193 fullCatalog = inStarObsCat
194 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
195 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
196 if not comp1
or not comp2:
197 raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema.")
199 fullCatalog = afwTable.BaseCatalog(outputSchema)
201 visitKey = outputSchema[
'visit'].asKey()
202 ccdKey = outputSchema[
'ccd'].asKey()
203 instMagKey = outputSchema[
'instMag'].asKey()
204 instMagErrKey = outputSchema[
'instMagErr'].asKey()
207 if self.config.doSubtractLocalBackground:
208 localBackgroundArea = np.pi*calibFluxApertureRadius**2.
215 for counter, visit
in enumerate(visitCat):
217 if visit[
'sources_read']:
220 expTime = visit[
'exptime']
222 dataRef = groupedDataRefs[visit[
'visit']][-1]
223 srcTable = dataRef.get()
225 df = srcTable.toDataFrame(columns)
227 goodSrc = self.sourceSelector.selectSources(df)
231 if self.config.doSubtractLocalBackground:
232 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
233 use, = np.where((goodSrc.selected) &
234 ((df[self.config.instFluxField].values - localBackground) > 0.0))
236 use, = np.where(goodSrc.selected)
238 tempCat = afwTable.BaseCatalog(fullCatalog.schema)
239 tempCat.resize(use.size)
241 tempCat[
'ra'][:] = np.deg2rad(df[
'ra'].values[use])
242 tempCat[
'dec'][:] = np.deg2rad(df[
'decl'].values[use])
243 tempCat[
'x'][:] = df[
'x'].values[use]
244 tempCat[
'y'][:] = df[
'y'].values[use]
245 tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
246 tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
247 tempCat[
'psf_candidate'] = df[
'Calib_psf_candidate'].values[use]
249 if self.config.doSubtractLocalBackground:
263 tempCat[
'deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] -
264 localBackground[use]) -
265 -2.5*np.log10(df[self.config.instFluxField].values[use]))
267 tempCat[
'deltaMagBkg'][:] = 0.0
270 for detector
in camera:
271 ccdId = detector.getId()
273 use2 = (tempCat[ccdKey] == ccdId)
274 tempCat[
'jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat[
'x'][use2],
276 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
277 visit[
'scaling'][ccdMapping[ccdId]])
278 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
282 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField +
'Err'].values[use] /
283 df[self.config.instFluxField].values[use])
286 if self.config.doApplyWcsJacobian:
287 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat[
'jacobian'][:])
289 fullCatalog.extend(tempCat)
292 with np.warnings.catch_warnings():
294 np.warnings.simplefilter(
"ignore")
296 instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
297 instMagErrIn = k*(df[self.config.apertureInnerInstFluxField +
'Err'].values[use] /
298 df[self.config.apertureInnerInstFluxField].values[use])
299 instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
300 instMagErrOut = k*(df[self.config.apertureOuterInstFluxField +
'Err'].values[use] /
301 df[self.config.apertureOuterInstFluxField].values[use])
303 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
304 np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
306 visit[
'deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
307 visit[
'sources_read'] =
True 309 self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
310 use.size, visit[
'visit'], visit[
'deltaAper'])
312 if ((counter % self.config.nVisitsPerCheckpoint) == 0
and 313 starObsDataRef
is not None and visitCatDataRef
is not None):
316 starObsDataRef.put(fullCatalog)
317 visitCatDataRef.put(visitCat)
319 self.log.info(
"Found all good star observations in %.2f s" %
320 (time.time() - startTime))
324 def _get_sourceTable_visit_columns(self):
326 Get the sourceTable_visit columns from the config. 331 List of columns to read from sourceTable_visit 333 columns = [self.config.visitDataRefName, self.config.ccdDataRefName,
334 'ra',
'decl',
'x',
'y', self.config.psfCandidateName,
335 self.config.instFluxField, self.config.instFluxField +
'Err',
336 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField +
'Err',
337 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField +
'Err']
338 if self.sourceSelector.config.doFlags:
339 columns.extend(self.sourceSelector.config.flags.bad)
340 if self.sourceSelector.config.doUnresolved:
341 columns.append(self.sourceSelector.config.unresolved.name)
342 if self.sourceSelector.config.doIsolated:
343 columns.append(self.sourceSelector.config.isolated.parentName)
344 columns.append(self.sourceSelector.config.isolated.nChildName)
345 if self.config.doSubtractLocalBackground:
346 columns.append(self.config.localBackgroundFluxField)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def findAndGroupDataRefs(self, butler, dataRefs)
apertureOuterInstFluxField
def computeApproxPixelAreaFields(camera)
doSubtractLocalBackground
def _makeSourceMapper(self, sourceSchema)
def _get_sourceTable_visit_columns(self)
apertureInnerInstFluxField