lsst.fgcmcal  8.5.1-hsc
fgcmBuildStarsTable.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 #
3 # This file is part of fgcmcal.
4 #
5 # Developed for the LSST Data Management System.
6 # This product includes software developed by the LSST Project
7 # (https://www.lsst.org).
8 # See the COPYRIGHT file at the top-level directory of this distribution
9 # for details of code ownership.
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the GNU General Public License
22 # along with this program. If not, see <https://www.gnu.org/licenses/>.
23 """Build star observations for input to FGCM using sourceTable_visit.
24 
25 This task finds all the visits and sourceTable_visits in a repository (or a
26 subset based on command line parameters) and extracts all the potential
27 calibration stars for input into fgcm. This task additionally uses fgcm to
28 match star observations into unique stars, and performs as much cleaning of the
29 input catalog as possible.
30 """
31 
32 import time
33 
34 import numpy as np
35 import collections
36 
37 import lsst.pex.config as pexConfig
38 import lsst.pipe.base as pipeBase
39 import lsst.afw.table as afwTable
40 
41 from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask
42 from .utilities import computeApproxPixelAreaFields
43 
44 __all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask']
45 
46 
48  """Config for FgcmBuildStarsTableTask"""
49 
50  referenceCCD = pexConfig.Field(
51  doc="Reference CCD for checking PSF and background",
52  dtype=int,
53  default=40,
54  )
55 
56  def setDefaults(self):
57  super().setDefaults()
58 
59  # The names here correspond to the post-transformed
60  # sourceTable_visit catalogs, which differ from the raw src
61  # catalogs. Therefore, all field and flag names cannot
62  # be derived from the base config class.
63  self.instFluxField = 'ApFlux_12_0_instFlux'
64  self.localBackgroundFluxField = 'LocalBackground_instFlux'
65  self.apertureInnerInstFluxField = 'ApFlux_12_0_instFlux'
66  self.apertureOuterInstFluxField = 'ApFlux_17_0_instFlux'
67  self.psfCandidateName = 'Calib_psf_candidate'
68 
69  sourceSelector = self.sourceSelector["science"]
70 
71  fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag'
72 
73  sourceSelector.flags.bad = ['PixelFlags_edge',
74  'PixelFlags_interpolatedCenter',
75  'PixelFlags_saturatedCenter',
76  'PixelFlags_crCenter',
77  'PixelFlags_bad',
78  'PixelFlags_interpolated',
79  'PixelFlags_saturated',
80  'Centroid_flag',
81  fluxFlagName]
82 
84  localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag'
85  sourceSelector.flags.bad.append(localBackgroundFlagName)
86 
87  sourceSelector.signalToNoise.fluxField = self.instFluxField
88  sourceSelector.signalToNoise.errField = self.instFluxField + 'Err'
89 
90  sourceSelector.isolated.parentName = 'parentSourceId'
91  sourceSelector.isolated.nChildName = 'Deblend_nChild'
92 
93  sourceSelector.unresolved.name = 'extendedness'
94 
95 
97  """
98  Build stars for the FGCM global calibration, using sourceTable_visit catalogs.
99  """
100  ConfigClass = FgcmBuildStarsTableConfig
101  RunnerClass = FgcmBuildStarsRunner
102  _DefaultName = "fgcmBuildStarsTable"
103 
104  @classmethod
105  def _makeArgumentParser(cls):
106  """Create an argument parser"""
107  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
108  parser.add_id_argument("--id", "sourceTable_visit", help="Data ID, e.g. --id visit=6789")
109 
110  return parser
111 
112  def findAndGroupDataRefs(self, butler, dataRefs):
113  self.log.info("Grouping dataRefs by %s" % (self.config.visitDataRefName))
114 
115  camera = butler.get('camera')
116 
117  ccdIds = []
118  for detector in camera:
119  ccdIds.append(detector.getId())
120  # Insert our preferred referenceCCD first:
121  # It is fine that this is listed twice, because we only need
122  # the first calexp that is found.
123  ccdIds.insert(0, self.config.referenceCCD)
124 
125  # The visitTable building code expects a dictionary of groupedDataRefs
126  # keyed by visit, the first element as the "primary" calexp dataRef.
127  # We then append the sourceTable_visit dataRef at the end for the
128  # code which does the data reading (fgcmMakeAllStarObservations).
129 
130  groupedDataRefs = collections.defaultdict(list)
131  for dataRef in dataRefs:
132  visit = dataRef.dataId[self.config.visitDataRefName]
133 
134  # Find an existing calexp (we need for psf and metadata)
135  # and make the relevant dataRef
136  for ccdId in ccdIds:
137  try:
138  calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit,
139  self.config.ccdDataRefName: ccdId})
140  except RuntimeError:
141  # Not found
142  continue
143 
144  # Make sure the dataset exists
145  if not calexpRef.datasetExists():
146  continue
147 
148  # It was found. Add and quit out, since we only
149  # need one calexp per visit.
150  groupedDataRefs[visit].append(calexpRef)
151  break
152 
153  # And append this dataRef
154  groupedDataRefs[visit].append(dataRef)
155 
156  return groupedDataRefs
157 
158  def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
159  calibFluxApertureRadius=None,
160  visitCatDataRef=None,
161  starObsDataRef=None,
162  inStarObsCat=None):
163  startTime = time.time()
164 
165  # If both dataRefs are None, then we assume the caller does not
166  # want to store checkpoint files. If both are set, we will
167  # do checkpoint files. And if only one is set, this is potentially
168  # unintentional and we will warn.
169  if (visitCatDataRef is not None and starObsDataRef is None or
170  visitCatDataRef is None and starObsDataRef is not None):
171  self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
172  "no checkpoint files will be persisted.")
173 
174  if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
175  raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
176 
177  # To get the correct output schema, we use similar code as fgcmBuildStarsTask
178  # We are not actually using this mapper, except to grab the outputSchema
179  dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
180  sourceSchema = dataRef.get('src_schema', immediate=True).schema
181  sourceMapper = self._makeSourceMapper(sourceSchema)
182  outputSchema = sourceMapper.getOutputSchema()
183 
184  # Construct mapping from ccd number to index
185  camera = dataRef.get('camera')
186  ccdMapping = {}
187  for ccdIndex, detector in enumerate(camera):
188  ccdMapping[detector.getId()] = ccdIndex
189 
190  approxPixelAreaFields = computeApproxPixelAreaFields(camera)
191 
192  if inStarObsCat is not None:
193  fullCatalog = inStarObsCat
194  comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
195  comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
196  if not comp1 or not comp2:
197  raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
198  else:
199  fullCatalog = afwTable.BaseCatalog(outputSchema)
200 
201  visitKey = outputSchema['visit'].asKey()
202  ccdKey = outputSchema['ccd'].asKey()
203  instMagKey = outputSchema['instMag'].asKey()
204  instMagErrKey = outputSchema['instMagErr'].asKey()
205 
206  # Prepare local background if desired
207  if self.config.doSubtractLocalBackground:
208  localBackgroundArea = np.pi*calibFluxApertureRadius**2.
209 
210  # Determine which columns we need from the sourceTable_visit catalogs
211  columns = self._get_sourceTable_visit_columns()
212 
213  k = 2.5/np.log(10.)
214 
215  for counter, visit in enumerate(visitCat):
216  # Check if these sources have already been read and stored in the checkpoint file
217  if visit['sources_read']:
218  continue
219 
220  expTime = visit['exptime']
221 
222  dataRef = groupedDataRefs[visit['visit']][-1]
223  srcTable = dataRef.get()
224 
225  df = srcTable.toDataFrame(columns)
226 
227  goodSrc = self.sourceSelector.selectSources(df)
228 
229  # Need to add a selection based on the local background correction
230  # if necessary
231  if self.config.doSubtractLocalBackground:
232  localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
233  use, = np.where((goodSrc.selected) &
234  ((df[self.config.instFluxField].values - localBackground) > 0.0))
235  else:
236  use, = np.where(goodSrc.selected)
237 
238  tempCat = afwTable.BaseCatalog(fullCatalog.schema)
239  tempCat.resize(use.size)
240 
241  tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
242  tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
243  tempCat['x'][:] = df['x'].values[use]
244  tempCat['y'][:] = df['y'].values[use]
245  tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
246  tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
247  tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use]
248 
249  if self.config.doSubtractLocalBackground:
250  # At the moment we only adjust the flux and not the flux
251  # error by the background because the error on
252  # base_LocalBackground_instFlux is the rms error in the
253  # background annulus, not the error on the mean in the
254  # background estimate (which is much smaller, by sqrt(n)
255  # pixels used to estimate the background, which we do not
256  # have access to in this task). In the default settings,
257  # the annulus is sufficiently large such that these
258  # additional errors are are negligibly small (much less
259  # than a mmag in quadrature).
260 
261  # This is the difference between the mag with local background correction
262  # and the mag without local background correction.
263  tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] -
264  localBackground[use]) -
265  -2.5*np.log10(df[self.config.instFluxField].values[use]))
266  else:
267  tempCat['deltaMagBkg'][:] = 0.0
268 
269  # Need to loop over ccds here
270  for detector in camera:
271  ccdId = detector.getId()
272  # used index for all observations with a given ccd
273  use2 = (tempCat[ccdKey] == ccdId)
274  tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
275  tempCat['y'][use2])
276  scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
277  visit['scaling'][ccdMapping[ccdId]])
278  tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
279 
280  # Compute instMagErr from instFluxErr/instFlux, any scaling
281  # will cancel out.
282  tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] /
283  df[self.config.instFluxField].values[use])
284 
285  # Apply the jacobian if configured
286  if self.config.doApplyWcsJacobian:
287  tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
288 
289  fullCatalog.extend(tempCat)
290 
291  # Now do the aperture information
292  with np.warnings.catch_warnings():
293  # Ignore warnings, we will filter infinites and nans below
294  np.warnings.simplefilter("ignore")
295 
296  instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
297  instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] /
298  df[self.config.apertureInnerInstFluxField].values[use])
299  instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
300  instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] /
301  df[self.config.apertureOuterInstFluxField].values[use])
302 
303  ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
304  np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
305 
306  visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
307  visit['sources_read'] = True
308 
309  self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
310  use.size, visit['visit'], visit['deltaAper'])
311 
312  if ((counter % self.config.nVisitsPerCheckpoint) == 0 and
313  starObsDataRef is not None and visitCatDataRef is not None):
314  # We need to persist both the stars and the visit catalog which gets
315  # additional metadata from each visit.
316  starObsDataRef.put(fullCatalog)
317  visitCatDataRef.put(visitCat)
318 
319  self.log.info("Found all good star observations in %.2f s" %
320  (time.time() - startTime))
321 
322  return fullCatalog
323 
324  def _get_sourceTable_visit_columns(self):
325  """
326  Get the sourceTable_visit columns from the config.
327 
328  Returns
329  -------
330  columns : `list`
331  List of columns to read from sourceTable_visit
332  """
333  columns = [self.config.visitDataRefName, self.config.ccdDataRefName,
334  'ra', 'decl', 'x', 'y', self.config.psfCandidateName,
335  self.config.instFluxField, self.config.instFluxField + 'Err',
336  self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err',
337  self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err']
338  if self.sourceSelector.config.doFlags:
339  columns.extend(self.sourceSelector.config.flags.bad)
340  if self.sourceSelector.config.doUnresolved:
341  columns.append(self.sourceSelector.config.unresolved.name)
342  if self.sourceSelector.config.doIsolated:
343  columns.append(self.sourceSelector.config.isolated.parentName)
344  columns.append(self.sourceSelector.config.isolated.nChildName)
345  if self.config.doSubtractLocalBackground:
346  columns.append(self.config.localBackgroundFluxField)
347 
348  return columns
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None)
def computeApproxPixelAreaFields(camera)
Definition: utilities.py:479