lsst.pipe.tasks  8.5-hsc+2
calibrate.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 
24 from lsstDebug import getDebugFrame
25 import lsst.pex.config as pexConfig
26 import lsst.pipe.base as pipeBase
28 import lsst.afw.table as afwTable
29 from lsst.meas.astrom import AstrometryTask, displayAstrometry, denormalizeMatches
30 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
31 from lsst.obs.base import ExposureIdInfo
32 import lsst.daf.base as dafBase
33 from lsst.afw.math import BackgroundList
34 from lsst.afw.table import IdFactory, SourceTable
35 from lsst.meas.algorithms import SourceDetectionTask, ReferenceObjectLoader
36 from lsst.meas.base import (SingleFrameMeasurementTask,
37  ApplyApCorrTask,
38  CatalogCalculationTask)
39 from lsst.meas.deblender import SourceDeblendTask
40 from .fakes import BaseFakeSourcesTask
41 from .photoCal import PhotoCalTask
42 
43 __all__ = ["CalibrateConfig", "CalibrateTask"]
44 
45 
46 class CalibrateConnections(pipeBase.PipelineTaskConnections, dimensions=("instrument", "visit", "detector"),
47  defaultTemplates={}):
48 
49  icSourceSchema = cT.InitInput(
50  doc="Schema produced by characterize image task, used to initialize this task",
51  name="icSrc_schema",
52  storageClass="SourceCatalog",
53  multiple=True
54  )
55 
56  outputSchema = cT.InitOutput(
57  doc="Schema after CalibrateTask has been initialized",
58  name="src_schema",
59  storageClass="SourceCatalog",
60  multiple=True
61  )
62 
63  exposure = cT.Input(
64  doc="Input image to calibrate",
65  name="icExp",
66  storageClass="ExposureF",
67  dimensions=("instrument", "visit", "detector"),
68  )
69 
70  background = cT.Input(
71  doc="Backgrounds determined by characterize task",
72  name="icExpBackground",
73  storageClass="Background",
74  dimensions=("instrument", "visit", "detector"),
75  )
76 
77  icSourceCat = cT.Input(
78  doc="Source catalog created by characterize task",
79  name="icSrc",
80  storageClass="SourceCatalog",
81  dimensions=("instrument", "visit", "detector"),
82  )
83 
84  astromRefCat = cT.PrerequisiteInput(
85  doc="Reference catalog to use for astrometry",
86  name="cal_ref_cat",
87  storageClass="SimpleCatalog",
88  dimensions=("skypix",),
89  deferLoad=True,
90  multiple=True,
91  )
92 
93  photoRefCat = cT.PrerequisiteInput(
94  doc="Reference catalog to use for photometric calibration",
95  name="cal_ref_cat",
96  storageClass="SimpleCatalog",
97  dimensions=("skypix",),
98  deferLoad=True,
99  multiple=True
100  )
101 
102  outputExposure = cT.Output(
103  doc="Exposure after running calibration task",
104  name="calexp",
105  storageClass="ExposureF",
106  dimensions=("instrument", "visit", "detector"),
107  )
108 
109  outputCat = cT.Output(
110  doc="Source catalog produced in calibrate task",
111  name="src",
112  storageClass="SourceCatalog",
113  dimensions=("instrument", "visit", "detector"),
114  )
115 
116  outputBackground = cT.Output(
117  doc="Background models estimated in calibration task",
118  name="calexpBackground",
119  storageClass="Background",
120  dimensions=("instrument", "visit", "detector"),
121  )
122 
123  matches = cT.Output(
124  doc="Source/refObj matches from the astrometry solver",
125  name="srcMatch",
126  storageClass="Catalog",
127  dimensions=("instrument", "visit", "detector"),
128  )
129 
130  matchesDenormalized = cT.Output(
131  doc="Denormalized matches from astrometry solver",
132  name="srcMatchFull",
133  storageClass="Catalog",
134  dimensions=("instrument", "visit", "detector"),
135  )
136 
137  def __init__(self, *, config=None):
138  super().__init__(config=config)
139  if config.doWriteMatches is False:
140  self.outputs.remove("matches")
141  if config.doWriteMatchesDenormalized is False:
142  self.outputs.remove("matchesDenormalized")
143 
144 
145 class CalibrateConfig(pipeBase.PipelineTaskConfig, pipelineConnections=CalibrateConnections):
146  """Config for CalibrateTask"""
147  doWrite = pexConfig.Field(
148  dtype=bool,
149  default=True,
150  doc="Save calibration results?",
151  )
152  doWriteHeavyFootprintsInSources = pexConfig.Field(
153  dtype=bool,
154  default=True,
155  doc="Include HeavyFootprint data in source table? If false then heavy "
156  "footprints are saved as normal footprints, which saves some space"
157  )
158  doWriteMatches = pexConfig.Field(
159  dtype=bool,
160  default=True,
161  doc="Write reference matches (ignored if doWrite false)?",
162  )
163  doWriteMatchesDenormalized = pexConfig.Field(
164  dtype=bool,
165  default=False,
166  doc=("Write reference matches in denormalized format? "
167  "This format uses more disk space, but is more convenient to "
168  "read. Ignored if doWriteMatches=False or doWrite=False."),
169  )
170  doAstrometry = pexConfig.Field(
171  dtype=bool,
172  default=True,
173  doc="Perform astrometric calibration?",
174  )
175  astromRefObjLoader = pexConfig.ConfigurableField(
176  target=LoadIndexedReferenceObjectsTask,
177  doc="reference object loader for astrometric calibration",
178  )
179  photoRefObjLoader = pexConfig.ConfigurableField(
180  target=LoadIndexedReferenceObjectsTask,
181  doc="reference object loader for photometric calibration",
182  )
183  astrometry = pexConfig.ConfigurableField(
184  target=AstrometryTask,
185  doc="Perform astrometric calibration to refine the WCS",
186  )
187  requireAstrometry = pexConfig.Field(
188  dtype=bool,
189  default=True,
190  doc=("Raise an exception if astrometry fails? Ignored if doAstrometry "
191  "false."),
192  )
193  doPhotoCal = pexConfig.Field(
194  dtype=bool,
195  default=True,
196  doc="Perform phometric calibration?",
197  )
198  requirePhotoCal = pexConfig.Field(
199  dtype=bool,
200  default=True,
201  doc=("Raise an exception if photoCal fails? Ignored if doPhotoCal "
202  "false."),
203  )
204  photoCal = pexConfig.ConfigurableField(
205  target=PhotoCalTask,
206  doc="Perform photometric calibration",
207  )
208  icSourceFieldsToCopy = pexConfig.ListField(
209  dtype=str,
210  default=("calib_psf_candidate", "calib_psf_used", "calib_psf_reserved"),
211  doc=("Fields to copy from the icSource catalog to the output catalog "
212  "for matching sources Any missing fields will trigger a "
213  "RuntimeError exception. Ignored if icSourceCat is not provided.")
214  )
215  matchRadiusPix = pexConfig.Field(
216  dtype=float,
217  default=3,
218  doc=("Match radius for matching icSourceCat objects to sourceCat "
219  "objects (pixels)"),
220  )
221  checkUnitsParseStrict = pexConfig.Field(
222  doc=("Strictness of Astropy unit compatibility check, can be 'raise', "
223  "'warn' or 'silent'"),
224  dtype=str,
225  default="raise",
226  )
227  detection = pexConfig.ConfigurableField(
228  target=SourceDetectionTask,
229  doc="Detect sources"
230  )
231  doDeblend = pexConfig.Field(
232  dtype=bool,
233  default=True,
234  doc="Run deblender input exposure"
235  )
236  deblend = pexConfig.ConfigurableField(
237  target=SourceDeblendTask,
238  doc="Split blended sources into their components"
239  )
240  measurement = pexConfig.ConfigurableField(
241  target=SingleFrameMeasurementTask,
242  doc="Measure sources"
243  )
244  doApCorr = pexConfig.Field(
245  dtype=bool,
246  default=True,
247  doc="Run subtask to apply aperture correction"
248  )
249  applyApCorr = pexConfig.ConfigurableField(
250  target=ApplyApCorrTask,
251  doc="Subtask to apply aperture corrections"
252  )
253  # If doApCorr is False, and the exposure does not have apcorrections
254  # already applied, the active plugins in catalogCalculation almost
255  # certainly should not contain the characterization plugin
256  catalogCalculation = pexConfig.ConfigurableField(
257  target=CatalogCalculationTask,
258  doc="Subtask to run catalogCalculation plugins on catalog"
259  )
260  doInsertFakes = pexConfig.Field(
261  dtype=bool,
262  default=False,
263  doc="Run fake sources injection task"
264  )
265  insertFakes = pexConfig.ConfigurableField(
266  target=BaseFakeSourcesTask,
267  doc="Injection of fake sources for testing purposes (must be "
268  "retargeted)"
269  )
270  doWriteExposure = pexConfig.Field(
271  dtype=bool,
272  default=True,
273  doc="Write the calexp? If fakes have been added then we do not want to write out the calexp as a "
274  "normal calexp but as a fakes_calexp."
275  )
276 
277  def setDefaults(self):
278  super().setDefaults()
279  self.detection.doTempLocalBackground = False
280  self.deblend.maxFootprintSize = 2000
281 
282  def validate(self):
283  super().validate()
284  astromRefCatGen2 = getattr(self.astromRefObjLoader, "ref_dataset_name", None)
285  if astromRefCatGen2 is not None and astromRefCatGen2 != self.connections.astromRefCat:
286  raise ValueError(
287  f"Gen2 ({astromRefCatGen2}) and Gen3 ({self.connections.astromRefCat}) astrometry reference "
288  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
289  )
290  photoRefCatGen2 = getattr(self.photoRefObjLoader, "ref_dataset_name", None)
291  if photoRefCatGen2 is not None and photoRefCatGen2 != self.connections.photoRefCat:
292  raise ValueError(
293  f"Gen2 ({photoRefCatGen2}) and Gen3 ({self.connections.photoRefCat}) photometry reference "
294  f"catalogs are different. These options must be kept in sync until Gen2 is retired."
295  )
296 
297 
298 
304 
305 class CalibrateTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
306  r"""!Calibrate an exposure: measure sources and perform astrometric and
307  photometric calibration
308 
309  @anchor CalibrateTask_
310 
311  @section pipe_tasks_calibrate_Contents Contents
312 
313  - @ref pipe_tasks_calibrate_Purpose
314  - @ref pipe_tasks_calibrate_Initialize
315  - @ref pipe_tasks_calibrate_IO
316  - @ref pipe_tasks_calibrate_Config
317  - @ref pipe_tasks_calibrate_Metadata
318  - @ref pipe_tasks_calibrate_Debug
319 
320 
321  @section pipe_tasks_calibrate_Purpose Description
322 
323  Given an exposure with a good PSF model and aperture correction map
324  (e.g. as provided by @ref CharacterizeImageTask), perform the following
325  operations:
326  - Run detection and measurement
327  - Run astrometry subtask to fit an improved WCS
328  - Run photoCal subtask to fit the exposure's photometric zero-point
329 
330  @section pipe_tasks_calibrate_Initialize Task initialisation
331 
332  @copydoc \_\_init\_\_
333 
334  @section pipe_tasks_calibrate_IO Invoking the Task
335 
336  If you want this task to unpersist inputs or persist outputs, then call
337  the `runDataRef` method (a wrapper around the `run` method).
338 
339  If you already have the inputs unpersisted and do not want to persist the
340  output then it is more direct to call the `run` method:
341 
342  @section pipe_tasks_calibrate_Config Configuration parameters
343 
344  See @ref CalibrateConfig
345 
346  @section pipe_tasks_calibrate_Metadata Quantities set in exposure Metadata
347 
348  Exposure metadata
349  <dl>
350  <dt>MAGZERO_RMS <dd>MAGZERO's RMS == sigma reported by photoCal task
351  <dt>MAGZERO_NOBJ <dd>Number of stars used == ngood reported by photoCal
352  task
353  <dt>COLORTERM1 <dd>?? (always 0.0)
354  <dt>COLORTERM2 <dd>?? (always 0.0)
355  <dt>COLORTERM3 <dd>?? (always 0.0)
356  </dl>
357 
358  @section pipe_tasks_calibrate_Debug Debug variables
359 
360  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink
361  interface supports a flag
362  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug
363  for more about `debug.py`.
364 
365  CalibrateTask has a debug dictionary containing one key:
366  <dl>
367  <dt>calibrate
368  <dd>frame (an int; <= 0 to not display) in which to display the exposure,
369  sources and matches. See @ref lsst.meas.astrom.displayAstrometry for
370  the meaning of the various symbols.
371  </dl>
372 
373  For example, put something like:
374  @code{.py}
375  import lsstDebug
376  def DebugInfo(name):
377  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would
378  # call us recursively
379  if name == "lsst.pipe.tasks.calibrate":
380  di.display = dict(
381  calibrate = 1,
382  )
383 
384  return di
385 
386  lsstDebug.Info = DebugInfo
387  @endcode
388  into your `debug.py` file and run `calibrateTask.py` with the `--debug`
389  flag.
390 
391  Some subtasks may have their own debug variables; see individual Task
392  documentation.
393  """
394 
395  # Example description used to live here, removed 2-20-2017 as per
396  # https://jira.lsstcorp.org/browse/DM-9520
397 
398  ConfigClass = CalibrateConfig
399  _DefaultName = "calibrate"
400  RunnerClass = pipeBase.ButlerInitializedTaskRunner
401 
402  def __init__(self, butler=None, astromRefObjLoader=None,
403  photoRefObjLoader=None, icSourceSchema=None,
404  initInputs=None, **kwargs):
405  """!Construct a CalibrateTask
406 
407  @param[in] butler The butler is passed to the refObjLoader constructor
408  in case it is needed. Ignored if the refObjLoader argument
409  provides a loader directly.
410  @param[in] astromRefObjLoader An instance of LoadReferenceObjectsTasks
411  that supplies an external reference catalog for astrometric
412  calibration. May be None if the desired loader can be constructed
413  from the butler argument or all steps requiring a reference catalog
414  are disabled.
415  @param[in] photoRefObjLoader An instance of LoadReferenceObjectsTasks
416  that supplies an external reference catalog for photometric
417  calibration. May be None if the desired loader can be constructed
418  from the butler argument or all steps requiring a reference catalog
419  are disabled.
420  @param[in] icSourceSchema schema for icSource catalog, or None.
421  Schema values specified in config.icSourceFieldsToCopy will be
422  taken from this schema. If set to None, no values will be
423  propagated from the icSourceCatalog
424  @param[in,out] kwargs other keyword arguments for
425  lsst.pipe.base.CmdLineTask
426  """
427  super().__init__(**kwargs)
428 
429  if icSourceSchema is None and butler is not None:
430  # Use butler to read icSourceSchema from disk.
431  icSourceSchema = butler.get("icSrc_schema", immediate=True).schema
432 
433  if icSourceSchema is None and butler is None and initInputs is not None:
434  icSourceSchema = initInputs['icSourceSchema'].schema
435 
436  if icSourceSchema is not None:
437  # use a schema mapper to avoid copying each field separately
438  self.schemaMapper = afwTable.SchemaMapper(icSourceSchema)
439  minimumSchema = afwTable.SourceTable.makeMinimalSchema()
440  self.schemaMapper.addMinimalSchema(minimumSchema, False)
441 
442  # Add fields to copy from an icSource catalog
443  # and a field to indicate that the source matched a source in that
444  # catalog. If any fields are missing then raise an exception, but
445  # first find all missing fields in order to make the error message
446  # more useful.
447  self.calibSourceKey = self.schemaMapper.addOutputField(
448  afwTable.Field["Flag"]("calib_detected",
449  "Source was detected as an icSource"))
450  missingFieldNames = []
451  for fieldName in self.config.icSourceFieldsToCopy:
452  try:
453  schemaItem = icSourceSchema.find(fieldName)
454  except Exception:
455  missingFieldNames.append(fieldName)
456  else:
457  # field found; if addMapping fails then raise an exception
458  self.schemaMapper.addMapping(schemaItem.getKey())
459 
460  if missingFieldNames:
461  raise RuntimeError("isSourceCat is missing fields {} "
462  "specified in icSourceFieldsToCopy"
463  .format(missingFieldNames))
464 
465  # produce a temporary schema to pass to the subtasks; finalize it
466  # later
467  self.schema = self.schemaMapper.editOutputSchema()
468  else:
469  self.schemaMapper = None
470  self.schema = afwTable.SourceTable.makeMinimalSchema()
471  self.makeSubtask('detection', schema=self.schema)
472 
473  self.algMetadata = dafBase.PropertyList()
474 
475  # Only create a subtask for fakes if configuration option is set
476  # N.B. the config for fake object task must be retargeted to a child
477  # of BaseFakeSourcesTask
478  if self.config.doInsertFakes:
479  self.makeSubtask("insertFakes")
480 
481  if self.config.doDeblend:
482  self.makeSubtask("deblend", schema=self.schema)
483  self.makeSubtask('measurement', schema=self.schema,
484  algMetadata=self.algMetadata)
485  if self.config.doApCorr:
486  self.makeSubtask('applyApCorr', schema=self.schema)
487  self.makeSubtask('catalogCalculation', schema=self.schema)
488 
489  if self.config.doAstrometry:
490  if astromRefObjLoader is None and butler is not None:
491  self.makeSubtask('astromRefObjLoader', butler=butler)
492  astromRefObjLoader = self.astromRefObjLoader
493  self.pixelMargin = astromRefObjLoader.config.pixelMargin
494  self.makeSubtask("astrometry", refObjLoader=astromRefObjLoader,
495  schema=self.schema)
496  if self.config.doPhotoCal:
497  if photoRefObjLoader is None and butler is not None:
498  self.makeSubtask('photoRefObjLoader', butler=butler)
499  photoRefObjLoader = self.photoRefObjLoader
500  self.pixelMargin = photoRefObjLoader.config.pixelMargin
501  self.makeSubtask("photoCal", refObjLoader=photoRefObjLoader,
502  schema=self.schema)
503 
504  if initInputs is not None and (astromRefObjLoader is not None or photoRefObjLoader is not None):
505  raise RuntimeError("PipelineTask form of this task should not be initialized with "
506  "reference object loaders.")
507 
508  if self.schemaMapper is not None:
509  # finalize the schema
510  self.schema = self.schemaMapper.getOutputSchema()
511  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
512 
513  sourceCatSchema = afwTable.SourceCatalog(self.schema)
514  sourceCatSchema.getTable().setMetadata(self.algMetadata)
515  self.outputSchema = sourceCatSchema
516 
517  @pipeBase.timeMethod
518  def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None,
519  doUnpersist=True):
520  """!Calibrate an exposure, optionally unpersisting inputs and
521  persisting outputs.
522 
523  This is a wrapper around the `run` method that unpersists inputs
524  (if `doUnpersist` true) and persists outputs (if `config.doWrite` true)
525 
526  @param[in] dataRef butler data reference corresponding to a science
527  image
528  @param[in,out] exposure characterized exposure (an
529  lsst.afw.image.ExposureF or similar), or None to unpersist existing
530  icExp and icBackground. See `run` method for details of what is
531  read and written.
532  @param[in,out] background initial model of background already
533  subtracted from exposure (an lsst.afw.math.BackgroundList). May be
534  None if no background has been subtracted, though that is unusual
535  for calibration. A refined background model is output. Ignored if
536  exposure is None.
537  @param[in] icSourceCat catalog from which to copy the fields specified
538  by icSourceKeys, or None;
539  @param[in] doUnpersist unpersist data:
540  - if True, exposure, background and icSourceCat are read from
541  dataRef and those three arguments must all be None;
542  - if False the exposure must be provided; background and
543  icSourceCat are optional. True is intended for running as a
544  command-line task, False for running as a subtask
545  @return same data as the calibrate method
546  """
547  self.log.info("Processing %s" % (dataRef.dataId))
548 
549  if doUnpersist:
550  if any(item is not None for item in (exposure, background,
551  icSourceCat)):
552  raise RuntimeError("doUnpersist true; exposure, background "
553  "and icSourceCat must all be None")
554  exposure = dataRef.get("icExp", immediate=True)
555  background = dataRef.get("icExpBackground", immediate=True)
556  icSourceCat = dataRef.get("icSrc", immediate=True)
557  elif exposure is None:
558  raise RuntimeError("doUnpersist false; exposure must be provided")
559 
560  exposureIdInfo = dataRef.get("expIdInfo")
561 
562  calRes = self.run(
563  exposure=exposure,
564  exposureIdInfo=exposureIdInfo,
565  background=background,
566  icSourceCat=icSourceCat,
567  )
568 
569  if self.config.doWrite:
570  self.writeOutputs(
571  dataRef=dataRef,
572  exposure=calRes.exposure,
573  background=calRes.background,
574  sourceCat=calRes.sourceCat,
575  astromMatches=calRes.astromMatches,
576  matchMeta=calRes.matchMeta,
577  )
578 
579  return calRes
580 
581  def runQuantum(self, butlerQC, inputRefs, outputRefs):
582  inputs = butlerQC.get(inputRefs)
583  expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
584  returnMaxBits=True)
585  inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
586 
587  if self.config.doAstrometry:
588  refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
589  for ref in inputRefs.astromRefCat],
590  refCats=inputs.pop('astromRefCat'),
591  config=self.config.astromRefObjLoader, log=self.log)
592  self.pixelMargin = refObjLoader.config.pixelMargin
593  self.astrometry.setRefObjLoader(refObjLoader)
594 
595  if self.config.doPhotoCal:
596  photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
597  for ref in inputRefs.photoRefCat],
598  refCats=inputs.pop('photoRefCat'),
599  config=self.config.photoRefObjLoader,
600  log=self.log)
601  self.pixelMargin = photoRefObjLoader.config.pixelMargin
602  self.photoCal.match.setRefObjLoader(photoRefObjLoader)
603 
604  outputs = self.run(**inputs)
605 
606  if self.config.doWriteMatches:
607  normalizedMatches = afwTable.packMatches(outputs.astromMatches)
608  normalizedMatches.table.setMetadata(outputs.matchMeta)
609  if self.config.doWriteMatchesDenormalized:
610  denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
611  outputs.matchesDenormalized = denormMatches
612  outputs.matches = normalizedMatches
613  butlerQC.put(outputs, outputRefs)
614 
615  def run(self, exposure, exposureIdInfo=None, background=None,
616  icSourceCat=None):
617  """!Calibrate an exposure (science image or coadd)
618 
619  @param[in,out] exposure exposure to calibrate (an
620  lsst.afw.image.ExposureF or similar);
621  in:
622  - MaskedImage
623  - Psf
624  out:
625  - MaskedImage has background subtracted
626  - Wcs is replaced
627  - PhotoCalib is replaced
628  @param[in] exposureIdInfo ID info for exposure (an
629  lsst.obs.base.ExposureIdInfo) If not provided, returned
630  SourceCatalog IDs will not be globally unique.
631  @param[in,out] background background model already subtracted from
632  exposure (an lsst.afw.math.BackgroundList). May be None if no
633  background has been subtracted, though that is unusual for
634  calibration. A refined background model is output.
635  @param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
636  from which we can copy some fields.
637 
638  @return pipe_base Struct containing these fields:
639  - exposure calibrate science exposure with refined WCS and PhotoCalib
640  - background model of background subtracted from exposure (an
641  lsst.afw.math.BackgroundList)
642  - sourceCat catalog of measured sources
643  - astromMatches list of source/refObj matches from the astrometry
644  solver
645  """
646  # detect, deblend and measure sources
647  if exposureIdInfo is None:
648  exposureIdInfo = ExposureIdInfo()
649 
650  if background is None:
651  background = BackgroundList()
652  sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
653  exposureIdInfo.unusedBits)
654  table = SourceTable.make(self.schema, sourceIdFactory)
655  table.setMetadata(self.algMetadata)
656 
657  detRes = self.detection.run(table=table, exposure=exposure,
658  doSmooth=True)
659  sourceCat = detRes.sources
660  if detRes.fpSets.background:
661  for bg in detRes.fpSets.background:
662  background.append(bg)
663  if self.config.doDeblend:
664  self.deblend.run(exposure=exposure, sources=sourceCat)
665  self.measurement.run(
666  measCat=sourceCat,
667  exposure=exposure,
668  exposureId=exposureIdInfo.expId
669  )
670  if self.config.doApCorr:
671  self.applyApCorr.run(
672  catalog=sourceCat,
673  apCorrMap=exposure.getInfo().getApCorrMap()
674  )
675  self.catalogCalculation.run(sourceCat)
676 
677  if icSourceCat is not None and \
678  len(self.config.icSourceFieldsToCopy) > 0:
679  self.copyIcSourceFields(icSourceCat=icSourceCat,
680  sourceCat=sourceCat)
681 
682  # TODO DM-11568: this contiguous check-and-copy could go away if we
683  # reserve enough space during SourceDetection and/or SourceDeblend.
684  # NOTE: sourceSelectors require contiguous catalogs, so ensure
685  # contiguity now, so views are preserved from here on.
686  if not sourceCat.isContiguous():
687  sourceCat = sourceCat.copy(deep=True)
688 
689  # perform astrometry calibration:
690  # fit an improved WCS and update the exposure's WCS in place
691  astromMatches = None
692  matchMeta = None
693  if self.config.doAstrometry:
694  try:
695  astromRes = self.astrometry.run(
696  exposure=exposure,
697  sourceCat=sourceCat,
698  )
699  astromMatches = astromRes.matches
700  matchMeta = astromRes.matchMeta
701  except Exception as e:
702  if self.config.requireAstrometry:
703  raise
704  self.log.warn("Unable to perform astrometric calibration "
705  "(%s): attempting to proceed" % e)
706 
707  # compute photometric calibration
708  if self.config.doPhotoCal:
709  try:
710  photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
711  exposure.setPhotoCalib(photoRes.photoCalib)
712  # TODO: reword this to phrase it in terms of the calibration factor?
713  self.log.info("Photometric zero-point: %f" %
714  photoRes.photoCalib.instFluxToMagnitude(1.0))
715  self.setMetadata(exposure=exposure, photoRes=photoRes)
716  except Exception as e:
717  if self.config.requirePhotoCal:
718  raise
719  self.log.warn("Unable to perform photometric calibration "
720  "(%s): attempting to proceed" % e)
721  self.setMetadata(exposure=exposure, photoRes=None)
722 
723  if self.config.doInsertFakes:
724  self.insertFakes.run(exposure, background=background)
725 
726  table = SourceTable.make(self.schema, sourceIdFactory)
727  table.setMetadata(self.algMetadata)
728 
729  detRes = self.detection.run(table=table, exposure=exposure,
730  doSmooth=True)
731  sourceCat = detRes.sources
732  if detRes.fpSets.background:
733  for bg in detRes.fpSets.background:
734  background.append(bg)
735  if self.config.doDeblend:
736  self.deblend.run(exposure=exposure, sources=sourceCat)
737  self.measurement.run(
738  measCat=sourceCat,
739  exposure=exposure,
740  exposureId=exposureIdInfo.expId
741  )
742  if self.config.doApCorr:
743  self.applyApCorr.run(
744  catalog=sourceCat,
745  apCorrMap=exposure.getInfo().getApCorrMap()
746  )
747  self.catalogCalculation.run(sourceCat)
748 
749  if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
750  self.copyIcSourceFields(icSourceCat=icSourceCat,
751  sourceCat=sourceCat)
752 
753  frame = getDebugFrame(self._display, "calibrate")
754  if frame:
755  displayAstrometry(
756  sourceCat=sourceCat,
757  exposure=exposure,
758  matches=astromMatches,
759  frame=frame,
760  pause=False,
761  )
762 
763  return pipeBase.Struct(
764  exposure=exposure,
765  background=background,
766  sourceCat=sourceCat,
767  astromMatches=astromMatches,
768  matchMeta=matchMeta,
769  # These are duplicate entries with different names for use with
770  # gen3 middleware
771  outputExposure=exposure,
772  outputCat=sourceCat,
773  outputBackground=background,
774  )
775 
776  def writeOutputs(self, dataRef, exposure, background, sourceCat,
777  astromMatches, matchMeta):
778  """Write output data to the output repository
779 
780  @param[in] dataRef butler data reference corresponding to a science
781  image
782  @param[in] exposure exposure to write
783  @param[in] background background model for exposure
784  @param[in] sourceCat catalog of measured sources
785  @param[in] astromMatches list of source/refObj matches from the
786  astrometry solver
787  """
788  dataRef.put(sourceCat, "src")
789  if self.config.doWriteMatches and astromMatches is not None:
790  normalizedMatches = afwTable.packMatches(astromMatches)
791  normalizedMatches.table.setMetadata(matchMeta)
792  dataRef.put(normalizedMatches, "srcMatch")
793  if self.config.doWriteMatchesDenormalized:
794  denormMatches = denormalizeMatches(astromMatches, matchMeta)
795  dataRef.put(denormMatches, "srcMatchFull")
796  if self.config.doWriteExposure:
797  dataRef.put(exposure, "calexp")
798  dataRef.put(background, "calexpBackground")
799 
800  def getSchemaCatalogs(self):
801  """Return a dict of empty catalogs for each catalog dataset produced
802  by this task.
803  """
804  sourceCat = afwTable.SourceCatalog(self.schema)
805  sourceCat.getTable().setMetadata(self.algMetadata)
806  return {"src": sourceCat}
807 
808  def setMetadata(self, exposure, photoRes=None):
809  """!Set task and exposure metadata
810 
811  Logs a warning and continues if needed data is missing.
812 
813  @param[in,out] exposure exposure whose metadata is to be set
814  @param[in] photoRes results of running photoCal; if None then it was
815  not run
816  """
817  if photoRes is None:
818  return
819 
820  metadata = exposure.getMetadata()
821 
822  # convert zero-point to (mag/sec/adu) for task MAGZERO metadata
823  try:
824  exposureTime = exposure.getInfo().getVisitInfo().getExposureTime()
825  magZero = photoRes.zp - 2.5*math.log10(exposureTime)
826  except Exception:
827  self.log.warn("Could not set normalized MAGZERO in header: no "
828  "exposure time")
829  magZero = math.nan
830 
831  try:
832  metadata.set('MAGZERO', magZero)
833  metadata.set('MAGZERO_RMS', photoRes.sigma)
834  metadata.set('MAGZERO_NOBJ', photoRes.ngood)
835  metadata.set('COLORTERM1', 0.0)
836  metadata.set('COLORTERM2', 0.0)
837  metadata.set('COLORTERM3', 0.0)
838  except Exception as e:
839  self.log.warn("Could not set exposure metadata: %s" % (e,))
840 
841  def copyIcSourceFields(self, icSourceCat, sourceCat):
842  """!Match sources in icSourceCat and sourceCat and copy the specified fields
843 
844  @param[in] icSourceCat catalog from which to copy fields
845  @param[in,out] sourceCat catalog to which to copy fields
846 
847  The fields copied are those specified by `config.icSourceFieldsToCopy`
848  that actually exist in the schema. This was set up by the constructor
849  using self.schemaMapper.
850  """
851  if self.schemaMapper is None:
852  raise RuntimeError("To copy icSource fields you must specify "
853  "icSourceSchema nd icSourceKeys when "
854  "constructing this task")
855  if icSourceCat is None or sourceCat is None:
856  raise RuntimeError("icSourceCat and sourceCat must both be "
857  "specified")
858  if len(self.config.icSourceFieldsToCopy) == 0:
859  self.log.warn("copyIcSourceFields doing nothing because "
860  "icSourceFieldsToCopy is empty")
861  return
862 
863  mc = afwTable.MatchControl()
864  mc.findOnlyClosest = False # return all matched objects
865  matches = afwTable.matchXy(icSourceCat, sourceCat,
866  self.config.matchRadiusPix, mc)
867  if self.config.doDeblend:
868  deblendKey = sourceCat.schema["deblend_nChild"].asKey()
869  # if deblended, keep children
870  matches = [m for m in matches if m[1].get(deblendKey) == 0]
871 
872  # Because we had to allow multiple matches to handle parents, we now
873  # need to prune to the best matches
874  # closest matches as a dict of icSourceCat source ID:
875  # (icSourceCat source, sourceCat source, distance in pixels)
876  bestMatches = {}
877  for m0, m1, d in matches:
878  id0 = m0.getId()
879  match = bestMatches.get(id0)
880  if match is None or d <= match[2]:
881  bestMatches[id0] = (m0, m1, d)
882  matches = list(bestMatches.values())
883 
884  # Check that no sourceCat sources are listed twice (we already know
885  # that each match has a unique icSourceCat source ID, due to using
886  # that ID as the key in bestMatches)
887  numMatches = len(matches)
888  numUniqueSources = len(set(m[1].getId() for m in matches))
889  if numUniqueSources != numMatches:
890  self.log.warn("{} icSourceCat sources matched only {} sourceCat "
891  "sources".format(numMatches, numUniqueSources))
892 
893  self.log.info("Copying flags from icSourceCat to sourceCat for "
894  "%s sources" % (numMatches,))
895 
896  # For each match: set the calibSourceKey flag and copy the desired
897  # fields
898  for icSrc, src, d in matches:
899  src.setFlag(self.calibSourceKey, True)
900  # src.assign copies the footprint from icSrc, which we don't want
901  # (DM-407)
902  # so set icSrc's footprint to src's footprint before src.assign,
903  # then restore it
904  icSrcFootprint = icSrc.getFootprint()
905  try:
906  icSrc.setFootprint(src.getFootprint())
907  src.assign(icSrc, self.schemaMapper)
908  finally:
909  icSrc.setFootprint(icSrcFootprint)
def copyIcSourceFields(self, icSourceCat, sourceCat)
Match sources in icSourceCat and sourceCat and copy the specified fields.
Definition: calibrate.py:841
def __init__(self, butler=None, astromRefObjLoader=None, photoRefObjLoader=None, icSourceSchema=None, initInputs=None, kwargs)
Construct a CalibrateTask.
Definition: calibrate.py:404
def run(self, exposure, exposureIdInfo=None, background=None, icSourceCat=None)
Calibrate an exposure (science image or coadd)
Definition: calibrate.py:616
def writeOutputs(self, dataRef, exposure, background, sourceCat, astromMatches, matchMeta)
Definition: calibrate.py:777
def setMetadata(self, exposure, photoRes=None)
Set task and exposure metadata.
Definition: calibrate.py:808
def runDataRef(self, dataRef, exposure=None, background=None, icSourceCat=None, doUnpersist=True)
Calibrate an exposure, optionally unpersisting inputs and persisting outputs.
Definition: calibrate.py:519
Calibrate an exposure: measure sources and perform astrometric and photometric calibration.
Definition: calibrate.py:305
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: calibrate.py:581