lsst.ip.isr  8.0-hsc+3
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .straylight import StrayLightTask
54 from .vignette import VignetteTask
55 
56 
57 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
58 
59 
60 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
61  dimensions={"instrument", "visit", "detector"},
62  defaultTemplates={}):
63  ccdExposure = cT.PrerequisiteInput(
64  name="raw",
65  doc="Input exposure to process.",
66  storageClass="Exposure",
67  dimensions=["instrument", "visit", "detector"],
68  )
69  camera = cT.PrerequisiteInput(
70  name="camera",
71  storageClass="Camera",
72  doc="Input camera to construct complete exposures.",
73  dimensions=["instrument", "calibration_label"],
74  )
75  bias = cT.PrerequisiteInput(
76  name="bias",
77  doc="Input bias calibration.",
78  storageClass="ImageF",
79  dimensions=["instrument", "calibration_label", "detector"],
80  )
81  dark = cT.PrerequisiteInput(
82  name='dark',
83  doc="Input dark calibration.",
84  storageClass="ImageF",
85  dimensions=["instrument", "calibration_label", "detector"],
86  )
87  flat = cT.PrerequisiteInput(
88  name="flat",
89  doc="Input flat calibration.",
90  storageClass="MaskedImageF",
91  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
92  )
93  fringes = cT.PrerequisiteInput(
94  name="fringe",
95  doc="Input fringe calibration.",
96  storageClass="ExposureF",
97  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
98  )
99  strayLightData = cT.PrerequisiteInput(
100  name='yBackground',
101  doc="Input stray light calibration.",
102  storageClass="StrayLightData",
103  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
104  )
105  bfKernel = cT.PrerequisiteInput(
106  name='bfKernel',
107  doc="Input brighter-fatter kernel.",
108  storageClass="NumpyArray",
109  dimensions=["instrument", "calibration_label"],
110  )
111  newBFKernel = cT.PrerequisiteInput(
112  name='brighterFatterKernel',
113  doc="Newer complete kernel + gain solutions.",
114  storageClass="BrighterFatterKernel",
115  dimensions=["instrument", "calibration_label", "detector"],
116  )
117  defects = cT.PrerequisiteInput(
118  name='defects',
119  doc="Input defect tables.",
120  storageClass="DefectsList",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  opticsTransmission = cT.PrerequisiteInput(
124  name="transmission_optics",
125  storageClass="TransmissionCurve",
126  doc="Transmission curve due to the optics.",
127  dimensions=["instrument", "calibration_label"],
128  )
129  filterTransmission = cT.PrerequisiteInput(
130  name="transmission_filter",
131  storageClass="TransmissionCurve",
132  doc="Transmission curve due to the filter.",
133  dimensions=["instrument", "physical_filter", "calibration_label"],
134  )
135  sensorTransmission = cT.PrerequisiteInput(
136  name="transmission_sensor",
137  storageClass="TransmissionCurve",
138  doc="Transmission curve due to the sensor.",
139  dimensions=["instrument", "calibration_label", "detector"],
140  )
141  atmosphereTransmission = cT.PrerequisiteInput(
142  name="transmission_atmosphere",
143  storageClass="TransmissionCurve",
144  doc="Transmission curve due to the atmosphere.",
145  dimensions=["instrument"],
146  )
147  illumMaskedImage = cT.PrerequisiteInput(
148  name="illum",
149  doc="Input illumination correction.",
150  storageClass="MaskedImageF",
151  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
152  )
153 
154  outputExposure = cT.Output(
155  name='postISRCCD',
156  doc="Output ISR processed exposure.",
157  storageClass="ExposureF",
158  dimensions=["instrument", "visit", "detector"],
159  )
160  preInterpExposure = cT.Output(
161  name='preInterpISRCCD',
162  doc="Output ISR processed exposure, with pixels left uninterpolated.",
163  storageClass="ExposureF",
164  dimensions=["instrument", "visit", "detector"],
165  )
166  outputOssThumbnail = cT.Output(
167  name="OssThumb",
168  doc="Output Overscan-subtracted thumbnail image.",
169  storageClass="Thumbnail",
170  dimensions=["instrument", "visit", "detector"],
171  )
172  outputFlattenedThumbnail = cT.Output(
173  name="FlattenedThumb",
174  doc="Output flat-corrected thumbnail image.",
175  storageClass="Thumbnail",
176  dimensions=["instrument", "visit", "detector"],
177  )
178 
179  def __init__(self, *, config=None):
180  super().__init__(config=config)
181 
182  if config.doBias is not True:
183  self.prerequisiteInputs.discard("bias")
184  if config.doLinearize is not True:
185  self.prerequisiteInputs.discard("linearizer")
186  if config.doCrosstalk is not True:
187  self.prerequisiteInputs.discard("crosstalkSources")
188  if config.doBrighterFatter is not True:
189  self.prerequisiteInputs.discard("bfKernel")
190  self.prerequisiteInputs.discard("newBFKernel")
191  if config.doDefect is not True:
192  self.prerequisiteInputs.discard("defects")
193  if config.doDark is not True:
194  self.prerequisiteInputs.discard("dark")
195  if config.doFlat is not True:
196  self.prerequisiteInputs.discard("flat")
197  if config.doAttachTransmissionCurve is not True:
198  self.prerequisiteInputs.discard("opticsTransmission")
199  self.prerequisiteInputs.discard("filterTransmission")
200  self.prerequisiteInputs.discard("sensorTransmission")
201  self.prerequisiteInputs.discard("atmosphereTransmission")
202  if config.doUseOpticsTransmission is not True:
203  self.prerequisiteInputs.discard("opticsTransmission")
204  if config.doUseFilterTransmission is not True:
205  self.prerequisiteInputs.discard("filterTransmission")
206  if config.doUseSensorTransmission is not True:
207  self.prerequisiteInputs.discard("sensorTransmission")
208  if config.doUseAtmosphereTransmission is not True:
209  self.prerequisiteInputs.discard("atmosphereTransmission")
210  if config.doIlluminationCorrection is not True:
211  self.prerequisiteInputs.discard("illumMaskedImage")
212 
213  if config.doWrite is not True:
214  self.outputs.discard("outputExposure")
215  self.outputs.discard("preInterpExposure")
216  self.outputs.discard("outputFlattenedThumbnail")
217  self.outputs.discard("outputOssThumbnail")
218  if config.doSaveInterpPixels is not True:
219  self.outputs.discard("preInterpExposure")
220  if config.qa.doThumbnailOss is not True:
221  self.outputs.discard("outputOssThumbnail")
222  if config.qa.doThumbnailFlattened is not True:
223  self.outputs.discard("outputFlattenedThumbnail")
224 
225 
226 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
227  pipelineConnections=IsrTaskConnections):
228  """Configuration parameters for IsrTask.
229 
230  Items are grouped in the order in which they are executed by the task.
231  """
232  datasetType = pexConfig.Field(
233  dtype=str,
234  doc="Dataset type for input data; users will typically leave this alone, "
235  "but camera-specific ISR tasks will override it",
236  default="raw",
237  )
238 
239  fallbackFilterName = pexConfig.Field(
240  dtype=str,
241  doc="Fallback default filter name for calibrations.",
242  optional=True
243  )
244  useFallbackDate = pexConfig.Field(
245  dtype=bool,
246  doc="Pass observation date when using fallback filter.",
247  default=False,
248  )
249  expectWcs = pexConfig.Field(
250  dtype=bool,
251  default=True,
252  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
253  )
254  fwhm = pexConfig.Field(
255  dtype=float,
256  doc="FWHM of PSF in arcseconds.",
257  default=1.0,
258  )
259  qa = pexConfig.ConfigField(
260  dtype=isrQa.IsrQaConfig,
261  doc="QA related configuration options.",
262  )
263 
264  # Image conversion configuration
265  doConvertIntToFloat = pexConfig.Field(
266  dtype=bool,
267  doc="Convert integer raw images to floating point values?",
268  default=True,
269  )
270 
271  # Saturated pixel handling.
272  doSaturation = pexConfig.Field(
273  dtype=bool,
274  doc="Mask saturated pixels? NB: this is totally independent of the"
275  " interpolation option - this is ONLY setting the bits in the mask."
276  " To have them interpolated make sure doSaturationInterpolation=True",
277  default=True,
278  )
279  saturatedMaskName = pexConfig.Field(
280  dtype=str,
281  doc="Name of mask plane to use in saturation detection and interpolation",
282  default="SAT",
283  )
284  saturation = pexConfig.Field(
285  dtype=float,
286  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
287  default=float("NaN"),
288  )
289  growSaturationFootprintSize = pexConfig.Field(
290  dtype=int,
291  doc="Number of pixels by which to grow the saturation footprints",
292  default=1,
293  )
294 
295  # Suspect pixel handling.
296  doSuspect = pexConfig.Field(
297  dtype=bool,
298  doc="Mask suspect pixels?",
299  default=False,
300  )
301  suspectMaskName = pexConfig.Field(
302  dtype=str,
303  doc="Name of mask plane to use for suspect pixels",
304  default="SUSPECT",
305  )
306  numEdgeSuspect = pexConfig.Field(
307  dtype=int,
308  doc="Number of edge pixels to be flagged as untrustworthy.",
309  default=0,
310  )
311 
312  # Initial masking options.
313  doSetBadRegions = pexConfig.Field(
314  dtype=bool,
315  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
316  default=True,
317  )
318  badStatistic = pexConfig.ChoiceField(
319  dtype=str,
320  doc="How to estimate the average value for BAD regions.",
321  default='MEANCLIP',
322  allowed={
323  "MEANCLIP": "Correct using the (clipped) mean of good data",
324  "MEDIAN": "Correct using the median of the good data",
325  },
326  )
327 
328  # Overscan subtraction configuration.
329  doOverscan = pexConfig.Field(
330  dtype=bool,
331  doc="Do overscan subtraction?",
332  default=True,
333  )
334  overscanFitType = pexConfig.ChoiceField(
335  dtype=str,
336  doc="The method for fitting the overscan bias level.",
337  default='MEDIAN',
338  allowed={
339  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
340  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
341  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
342  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
343  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
344  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
345  "MEAN": "Correct using the mean of the overscan region",
346  "MEANCLIP": "Correct using a clipped mean of the overscan region",
347  "MEDIAN": "Correct using the median of the overscan region",
348  },
349  )
350  overscanOrder = pexConfig.Field(
351  dtype=int,
352  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
353  "or number of spline knots if overscan fit type is a spline."),
354  default=1,
355  )
356  overscanNumSigmaClip = pexConfig.Field(
357  dtype=float,
358  doc="Rejection threshold (sigma) for collapsing overscan before fit",
359  default=3.0,
360  )
361  overscanIsInt = pexConfig.Field(
362  dtype=bool,
363  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN",
364  default=True,
365  )
366  overscanNumLeadingColumnsToSkip = pexConfig.Field(
367  dtype=int,
368  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
369  default=0,
370  )
371  overscanNumTrailingColumnsToSkip = pexConfig.Field(
372  dtype=int,
373  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
374  default=0,
375  )
376  overscanMaxDev = pexConfig.Field(
377  dtype=float,
378  doc="Maximum deviation from the median for overscan",
379  default=1000.0, check=lambda x: x > 0
380  )
381  overscanBiasJump = pexConfig.Field(
382  dtype=bool,
383  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
384  default=False,
385  )
386  overscanBiasJumpKeyword = pexConfig.Field(
387  dtype=str,
388  doc="Header keyword containing information about devices.",
389  default="NO_SUCH_KEY",
390  )
391  overscanBiasJumpDevices = pexConfig.ListField(
392  dtype=str,
393  doc="List of devices that need piecewise overscan correction.",
394  default=(),
395  )
396  overscanBiasJumpLocation = pexConfig.Field(
397  dtype=int,
398  doc="Location of bias jump along y-axis.",
399  default=0,
400  )
401 
402  # Amplifier to CCD assembly configuration
403  doAssembleCcd = pexConfig.Field(
404  dtype=bool,
405  default=True,
406  doc="Assemble amp-level exposures into a ccd-level exposure?"
407  )
408  assembleCcd = pexConfig.ConfigurableField(
409  target=AssembleCcdTask,
410  doc="CCD assembly task",
411  )
412 
413  # General calibration configuration.
414  doAssembleIsrExposures = pexConfig.Field(
415  dtype=bool,
416  default=False,
417  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
418  )
419  doTrimToMatchCalib = pexConfig.Field(
420  dtype=bool,
421  default=False,
422  doc="Trim raw data to match calibration bounding boxes?"
423  )
424 
425  # Bias subtraction.
426  doBias = pexConfig.Field(
427  dtype=bool,
428  doc="Apply bias frame correction?",
429  default=True,
430  )
431  biasDataProductName = pexConfig.Field(
432  dtype=str,
433  doc="Name of the bias data product",
434  default="bias",
435  )
436 
437  # Variance construction
438  doVariance = pexConfig.Field(
439  dtype=bool,
440  doc="Calculate variance?",
441  default=True
442  )
443  gain = pexConfig.Field(
444  dtype=float,
445  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
446  default=float("NaN"),
447  )
448  readNoise = pexConfig.Field(
449  dtype=float,
450  doc="The read noise to use if no Detector is present in the Exposure",
451  default=0.0,
452  )
453  doEmpiricalReadNoise = pexConfig.Field(
454  dtype=bool,
455  default=False,
456  doc="Calculate empirical read noise instead of value from AmpInfo data?"
457  )
458 
459  # Linearization.
460  doLinearize = pexConfig.Field(
461  dtype=bool,
462  doc="Correct for nonlinearity of the detector's response?",
463  default=True,
464  )
465 
466  # Crosstalk.
467  doCrosstalk = pexConfig.Field(
468  dtype=bool,
469  doc="Apply intra-CCD crosstalk correction?",
470  default=False,
471  )
472  doCrosstalkBeforeAssemble = pexConfig.Field(
473  dtype=bool,
474  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
475  default=False,
476  )
477  crosstalk = pexConfig.ConfigurableField(
478  target=CrosstalkTask,
479  doc="Intra-CCD crosstalk correction",
480  )
481 
482  # Masking options.
483  doDefect = pexConfig.Field(
484  dtype=bool,
485  doc="Apply correction for CCD defects, e.g. hot pixels?",
486  default=True,
487  )
488  doNanMasking = pexConfig.Field(
489  dtype=bool,
490  doc="Mask NAN pixels?",
491  default=True,
492  )
493  doWidenSaturationTrails = pexConfig.Field(
494  dtype=bool,
495  doc="Widen bleed trails based on their width?",
496  default=True
497  )
498 
499  # Brighter-Fatter correction.
500  doBrighterFatter = pexConfig.Field(
501  dtype=bool,
502  default=False,
503  doc="Apply the brighter fatter correction"
504  )
505  brighterFatterLevel = pexConfig.ChoiceField(
506  dtype=str,
507  default="DETECTOR",
508  doc="The level at which to correct for brighter-fatter.",
509  allowed={
510  "AMP": "Every amplifier treated separately.",
511  "DETECTOR": "One kernel per detector",
512  }
513  )
514  brighterFatterMaxIter = pexConfig.Field(
515  dtype=int,
516  default=10,
517  doc="Maximum number of iterations for the brighter fatter correction"
518  )
519  brighterFatterThreshold = pexConfig.Field(
520  dtype=float,
521  default=1000,
522  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
523  " absolute value of the difference between the current corrected image and the one"
524  " from the previous iteration summed over all the pixels."
525  )
526  brighterFatterApplyGain = pexConfig.Field(
527  dtype=bool,
528  default=True,
529  doc="Should the gain be applied when applying the brighter fatter correction?"
530  )
531  brighterFatterMaskGrowSize = pexConfig.Field(
532  dtype=int,
533  default=0,
534  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
535  " when brighter-fatter correction is applied."
536  )
537 
538  # Dark subtraction.
539  doDark = pexConfig.Field(
540  dtype=bool,
541  doc="Apply dark frame correction?",
542  default=True,
543  )
544  darkDataProductName = pexConfig.Field(
545  dtype=str,
546  doc="Name of the dark data product",
547  default="dark",
548  )
549 
550  # Camera-specific stray light removal.
551  doStrayLight = pexConfig.Field(
552  dtype=bool,
553  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
554  default=False,
555  )
556  strayLight = pexConfig.ConfigurableField(
557  target=StrayLightTask,
558  doc="y-band stray light correction"
559  )
560 
561  # Flat correction.
562  doFlat = pexConfig.Field(
563  dtype=bool,
564  doc="Apply flat field correction?",
565  default=True,
566  )
567  flatDataProductName = pexConfig.Field(
568  dtype=str,
569  doc="Name of the flat data product",
570  default="flat",
571  )
572  flatScalingType = pexConfig.ChoiceField(
573  dtype=str,
574  doc="The method for scaling the flat on the fly.",
575  default='USER',
576  allowed={
577  "USER": "Scale by flatUserScale",
578  "MEAN": "Scale by the inverse of the mean",
579  "MEDIAN": "Scale by the inverse of the median",
580  },
581  )
582  flatUserScale = pexConfig.Field(
583  dtype=float,
584  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
585  default=1.0,
586  )
587  doTweakFlat = pexConfig.Field(
588  dtype=bool,
589  doc="Tweak flats to match observed amplifier ratios?",
590  default=False
591  )
592 
593  # Amplifier normalization based on gains instead of using flats configuration.
594  doApplyGains = pexConfig.Field(
595  dtype=bool,
596  doc="Correct the amplifiers for their gains instead of applying flat correction",
597  default=False,
598  )
599  normalizeGains = pexConfig.Field(
600  dtype=bool,
601  doc="Normalize all the amplifiers in each CCD to have the same median value.",
602  default=False,
603  )
604 
605  # Fringe correction.
606  doFringe = pexConfig.Field(
607  dtype=bool,
608  doc="Apply fringe correction?",
609  default=True,
610  )
611  fringe = pexConfig.ConfigurableField(
612  target=FringeTask,
613  doc="Fringe subtraction task",
614  )
615  fringeAfterFlat = pexConfig.Field(
616  dtype=bool,
617  doc="Do fringe subtraction after flat-fielding?",
618  default=True,
619  )
620 
621  # Distortion model application.
622  doAddDistortionModel = pexConfig.Field(
623  dtype=bool,
624  doc="Apply a distortion model based on camera geometry to the WCS?",
625  default=True,
626  deprecated=("Camera geometry is incorporated when reading the raw files."
627  " This option no longer is used, and will be removed after v19.")
628  )
629 
630  # Initial CCD-level background statistics options.
631  doMeasureBackground = pexConfig.Field(
632  dtype=bool,
633  doc="Measure the background level on the reduced image?",
634  default=False,
635  )
636 
637  # Camera-specific masking configuration.
638  doCameraSpecificMasking = pexConfig.Field(
639  dtype=bool,
640  doc="Mask camera-specific bad regions?",
641  default=False,
642  )
643  masking = pexConfig.ConfigurableField(
644  target=MaskingTask,
645  doc="Masking task."
646  )
647 
648  # Interpolation options.
649 
650  doInterpolate = pexConfig.Field(
651  dtype=bool,
652  doc="Interpolate masked pixels?",
653  default=True,
654  )
655  doSaturationInterpolation = pexConfig.Field(
656  dtype=bool,
657  doc="Perform interpolation over pixels masked as saturated?"
658  " NB: This is independent of doSaturation; if that is False this plane"
659  " will likely be blank, resulting in a no-op here.",
660  default=True,
661  )
662  doNanInterpolation = pexConfig.Field(
663  dtype=bool,
664  doc="Perform interpolation over pixels masked as NaN?"
665  " NB: This is independent of doNanMasking; if that is False this plane"
666  " will likely be blank, resulting in a no-op here.",
667  default=True,
668  )
669  doNanInterpAfterFlat = pexConfig.Field(
670  dtype=bool,
671  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
672  "also have to interpolate them before flat-fielding."),
673  default=False,
674  )
675  maskListToInterpolate = pexConfig.ListField(
676  dtype=str,
677  doc="List of mask planes that should be interpolated.",
678  default=['SAT', 'BAD', 'UNMASKEDNAN'],
679  )
680  doSaveInterpPixels = pexConfig.Field(
681  dtype=bool,
682  doc="Save a copy of the pre-interpolated pixel values?",
683  default=False,
684  )
685 
686  # Default photometric calibration options.
687  fluxMag0T1 = pexConfig.DictField(
688  keytype=str,
689  itemtype=float,
690  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
691  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
692  ))
693  )
694  defaultFluxMag0T1 = pexConfig.Field(
695  dtype=float,
696  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
697  default=pow(10.0, 0.4*28.0)
698  )
699 
700  # Vignette correction configuration.
701  doVignette = pexConfig.Field(
702  dtype=bool,
703  doc="Apply vignetting parameters?",
704  default=False,
705  )
706  vignette = pexConfig.ConfigurableField(
707  target=VignetteTask,
708  doc="Vignetting task.",
709  )
710 
711  # Transmission curve configuration.
712  doAttachTransmissionCurve = pexConfig.Field(
713  dtype=bool,
714  default=False,
715  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
716  )
717  doUseOpticsTransmission = pexConfig.Field(
718  dtype=bool,
719  default=True,
720  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
721  )
722  doUseFilterTransmission = pexConfig.Field(
723  dtype=bool,
724  default=True,
725  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
726  )
727  doUseSensorTransmission = pexConfig.Field(
728  dtype=bool,
729  default=True,
730  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
731  )
732  doUseAtmosphereTransmission = pexConfig.Field(
733  dtype=bool,
734  default=True,
735  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
736  )
737 
738  # Illumination correction.
739  doIlluminationCorrection = pexConfig.Field(
740  dtype=bool,
741  default=False,
742  doc="Perform illumination correction?"
743  )
744  illuminationCorrectionDataProductName = pexConfig.Field(
745  dtype=str,
746  doc="Name of the illumination correction data product.",
747  default="illumcor",
748  )
749  illumScale = pexConfig.Field(
750  dtype=float,
751  doc="Scale factor for the illumination correction.",
752  default=1.0,
753  )
754  illumFilters = pexConfig.ListField(
755  dtype=str,
756  default=[],
757  doc="Only perform illumination correction for these filters."
758  )
759 
760  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
761  doWrite = pexConfig.Field(
762  dtype=bool,
763  doc="Persist postISRCCD?",
764  default=True,
765  )
766 
767  def validate(self):
768  super().validate()
769  if self.doFlat and self.doApplyGains:
770  raise ValueError("You may not specify both doFlat and doApplyGains")
771  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
772  self.config.maskListToInterpolate.append("SAT")
773  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
774  self.config.maskListToInterpolate.append("UNMASKEDNAN")
775 
776 
777 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
778  """Apply common instrument signature correction algorithms to a raw frame.
779 
780  The process for correcting imaging data is very similar from
781  camera to camera. This task provides a vanilla implementation of
782  doing these corrections, including the ability to turn certain
783  corrections off if they are not needed. The inputs to the primary
784  method, `run()`, are a raw exposure to be corrected and the
785  calibration data products. The raw input is a single chip sized
786  mosaic of all amps including overscans and other non-science
787  pixels. The method `runDataRef()` identifies and defines the
788  calibration data products, and is intended for use by a
789  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
790  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
791  subclassed for different camera, although the most camera specific
792  methods have been split into subtasks that can be redirected
793  appropriately.
794 
795  The __init__ method sets up the subtasks for ISR processing, using
796  the defaults from `lsst.ip.isr`.
797 
798  Parameters
799  ----------
800  args : `list`
801  Positional arguments passed to the Task constructor. None used at this time.
802  kwargs : `dict`, optional
803  Keyword arguments passed on to the Task constructor. None used at this time.
804  """
805  ConfigClass = IsrTaskConfig
806  _DefaultName = "isr"
807 
808  def __init__(self, **kwargs):
809  super().__init__(**kwargs)
810  self.makeSubtask("assembleCcd")
811  self.makeSubtask("crosstalk")
812  self.makeSubtask("strayLight")
813  self.makeSubtask("fringe")
814  self.makeSubtask("masking")
815  self.makeSubtask("vignette")
816 
817  def runQuantum(self, butlerQC, inputRefs, outputRefs):
818  inputs = butlerQC.get(inputRefs)
819 
820  try:
821  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
822  except Exception as e:
823  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
824  (inputRefs, e))
825 
826  inputs['isGen3'] = True
827 
828  detector = inputs['ccdExposure'].getDetector()
829  if self.doLinearize(detector) is True:
830  if 'linearizer' not in inputs:
831  linearityName = detector.getAmplifiers()[0].getLinearityType()
832  inputs['linearizer'] = linearize.getLinearityTypeByName(linearityName)()
833 
834  if self.config.doDefect is True:
835  if "defects" in inputs and inputs['defects'] is not None:
836  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
837  # masking expects a list of defects defined by their bounding box
838  if not isinstance(inputs["defects"], Defects):
839  inputs["defects"] = Defects.fromTable(inputs["defects"])
840 
841  # Load the correct style of brighter fatter kernel, and repack
842  # the information as a numpy array.
843  if self.config.doBrighterFatter:
844  brighterFatterKernel = inputs.pop('newBFKernel', None)
845  if brighterFatterKernel is None:
846  brighterFatterKernel = inputs.get('bfKernel', None)
847 
848  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
849  detId = detector.getId()
850  inputs['bfGains'] = brighterFatterKernel.gain
851  # If the kernel is not an ndarray, it's the cp_pipe version
852  # so extract the kernel for this detector, or raise an error
853  if self.config.brighterFatterLevel == 'DETECTOR':
854  if brighterFatterKernel.detectorKernel:
855  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
856  elif brighterFatterKernel.detectorKernelFromAmpKernels:
857  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
858  else:
859  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
860  else:
861  # TODO DM-15631 for implementing this
862  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
863 
864  # Broken: DM-17169
865  # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only. This needs to be
866  # fixed for non-HSC cameras in the future.
867  # inputs['crosstalkSources'] = (self.crosstalk.prepCrosstalk(inputsIds['ccdExposure'])
868  # if self.config.doCrosstalk else None)
869 
870  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
871  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
872  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
873  expId=expId,
874  assembler=self.assembleCcd
875  if self.config.doAssembleIsrExposures else None)
876  else:
877  inputs['fringes'] = pipeBase.Struct(fringes=None)
878 
879  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
880  if 'strayLightData' not in inputs:
881  inputs['strayLightData'] = None
882 
883  outputs = self.run(**inputs)
884  butlerQC.put(outputs, outputRefs)
885 
886  def readIsrData(self, dataRef, rawExposure):
887  """!Retrieve necessary frames for instrument signature removal.
888 
889  Pre-fetching all required ISR data products limits the IO
890  required by the ISR. Any conflict between the calibration data
891  available and that needed for ISR is also detected prior to
892  doing processing, allowing it to fail quickly.
893 
894  Parameters
895  ----------
896  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
897  Butler reference of the detector data to be processed
898  rawExposure : `afw.image.Exposure`
899  The raw exposure that will later be corrected with the
900  retrieved calibration data; should not be modified in this
901  method.
902 
903  Returns
904  -------
905  result : `lsst.pipe.base.Struct`
906  Result struct with components (which may be `None`):
907  - ``bias``: bias calibration frame (`afw.image.Exposure`)
908  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
909  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
910  - ``dark``: dark calibration frame (`afw.image.Exposure`)
911  - ``flat``: flat calibration frame (`afw.image.Exposure`)
912  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
913  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
914  - ``fringes``: `lsst.pipe.base.Struct` with components:
915  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
916  - ``seed``: random seed derived from the ccdExposureId for random
917  number generator (`uint32`).
918  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
919  A ``TransmissionCurve`` that represents the throughput of the optics,
920  to be evaluated in focal-plane coordinates.
921  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
922  A ``TransmissionCurve`` that represents the throughput of the filter
923  itself, to be evaluated in focal-plane coordinates.
924  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
925  A ``TransmissionCurve`` that represents the throughput of the sensor
926  itself, to be evaluated in post-assembly trimmed detector coordinates.
927  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
928  A ``TransmissionCurve`` that represents the throughput of the
929  atmosphere, assumed to be spatially constant.
930  - ``strayLightData`` : `object`
931  An opaque object containing calibration information for
932  stray-light correction. If `None`, no correction will be
933  performed.
934  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
935 
936  Raises
937  ------
938  NotImplementedError :
939  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
940  """
941  try:
942  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
943  dateObs = dateObs.toPython().isoformat()
944  except RuntimeError:
945  self.log.warn("Unable to identify dateObs for rawExposure.")
946  dateObs = None
947 
948  ccd = rawExposure.getDetector()
949  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
950  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
951  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
952  if self.config.doBias else None)
953  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
954  linearizer = (dataRef.get("linearizer", immediate=True)
955  if self.doLinearize(ccd) else None)
956  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef)
957  if self.config.doCrosstalk else None)
958  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
959  if self.config.doDark else None)
960  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
961  dateObs=dateObs)
962  if self.config.doFlat else None)
963 
964  brighterFatterKernel = None
965  brighterFatterGains = None
966  if self.config.doBrighterFatter is True:
967  try:
968  # Use the new-style cp_pipe version of the kernel if it exists
969  # If using a new-style kernel, always use the self-consistent
970  # gains, i.e. the ones inside the kernel object itself
971  brighterFatterKernel = dataRef.get("brighterFatterKernel")
972  brighterFatterGains = brighterFatterKernel.gain
973  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
974  except NoResults:
975  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
976  brighterFatterKernel = dataRef.get("bfKernel")
977  self.log.info("Old style bright-fatter kernel (np.array) loaded")
978  except NoResults:
979  brighterFatterKernel = None
980  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
981  # If the kernel is not an ndarray, it's the cp_pipe version
982  # so extract the kernel for this detector, or raise an error
983  if self.config.brighterFatterLevel == 'DETECTOR':
984  if brighterFatterKernel.detectorKernel:
985  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
986  elif brighterFatterKernel.detectorKernelFromAmpKernels:
987  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
988  else:
989  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
990  else:
991  # TODO DM-15631 for implementing this
992  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
993 
994  defectList = (dataRef.get("defects")
995  if self.config.doDefect else None)
996  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
997  if self.config.doAssembleIsrExposures else None)
998  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
999  else pipeBase.Struct(fringes=None))
1000 
1001  if self.config.doAttachTransmissionCurve:
1002  opticsTransmission = (dataRef.get("transmission_optics")
1003  if self.config.doUseOpticsTransmission else None)
1004  filterTransmission = (dataRef.get("transmission_filter")
1005  if self.config.doUseFilterTransmission else None)
1006  sensorTransmission = (dataRef.get("transmission_sensor")
1007  if self.config.doUseSensorTransmission else None)
1008  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1009  if self.config.doUseAtmosphereTransmission else None)
1010  else:
1011  opticsTransmission = None
1012  filterTransmission = None
1013  sensorTransmission = None
1014  atmosphereTransmission = None
1015 
1016  if self.config.doStrayLight:
1017  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1018  else:
1019  strayLightData = None
1020 
1021  illumMaskedImage = (self.getIsrExposure(dataRef,
1022  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1023  if (self.config.doIlluminationCorrection and
1024  filterName in self.config.illumFilters)
1025  else None)
1026 
1027  # Struct should include only kwargs to run()
1028  return pipeBase.Struct(bias=biasExposure,
1029  linearizer=linearizer,
1030  crosstalkSources=crosstalkSources,
1031  dark=darkExposure,
1032  flat=flatExposure,
1033  bfKernel=brighterFatterKernel,
1034  bfGains=brighterFatterGains,
1035  defects=defectList,
1036  fringes=fringeStruct,
1037  opticsTransmission=opticsTransmission,
1038  filterTransmission=filterTransmission,
1039  sensorTransmission=sensorTransmission,
1040  atmosphereTransmission=atmosphereTransmission,
1041  strayLightData=strayLightData,
1042  illumMaskedImage=illumMaskedImage
1043  )
1044 
1045  @pipeBase.timeMethod
1046  def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
1047  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1048  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1049  sensorTransmission=None, atmosphereTransmission=None,
1050  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1051  isGen3=False,
1052  ):
1053  """!Perform instrument signature removal on an exposure.
1054 
1055  Steps included in the ISR processing, in order performed, are:
1056  - saturation and suspect pixel masking
1057  - overscan subtraction
1058  - CCD assembly of individual amplifiers
1059  - bias subtraction
1060  - variance image construction
1061  - linearization of non-linear response
1062  - crosstalk masking
1063  - brighter-fatter correction
1064  - dark subtraction
1065  - fringe correction
1066  - stray light subtraction
1067  - flat correction
1068  - masking of known defects and camera specific features
1069  - vignette calculation
1070  - appending transmission curve and distortion model
1071 
1072  Parameters
1073  ----------
1074  ccdExposure : `lsst.afw.image.Exposure`
1075  The raw exposure that is to be run through ISR. The
1076  exposure is modified by this method.
1077  camera : `lsst.afw.cameraGeom.Camera`, optional
1078  The camera geometry for this exposure. Used to select the
1079  distortion model appropriate for this data.
1080  bias : `lsst.afw.image.Exposure`, optional
1081  Bias calibration frame.
1082  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1083  Functor for linearization.
1084  crosstalkSources : `list`, optional
1085  List of possible crosstalk sources.
1086  dark : `lsst.afw.image.Exposure`, optional
1087  Dark calibration frame.
1088  flat : `lsst.afw.image.Exposure`, optional
1089  Flat calibration frame.
1090  bfKernel : `numpy.ndarray`, optional
1091  Brighter-fatter kernel.
1092  bfGains : `dict` of `float`, optional
1093  Gains used to override the detector's nominal gains for the
1094  brighter-fatter correction. A dict keyed by amplifier name for
1095  the detector in question.
1096  defects : `lsst.meas.algorithms.Defects`, optional
1097  List of defects.
1098  fringes : `lsst.pipe.base.Struct`, optional
1099  Struct containing the fringe correction data, with
1100  elements:
1101  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1102  - ``seed``: random seed derived from the ccdExposureId for random
1103  number generator (`uint32`)
1104  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1105  A ``TransmissionCurve`` that represents the throughput of the optics,
1106  to be evaluated in focal-plane coordinates.
1107  filterTransmission : `lsst.afw.image.TransmissionCurve`
1108  A ``TransmissionCurve`` that represents the throughput of the filter
1109  itself, to be evaluated in focal-plane coordinates.
1110  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1111  A ``TransmissionCurve`` that represents the throughput of the sensor
1112  itself, to be evaluated in post-assembly trimmed detector coordinates.
1113  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1114  A ``TransmissionCurve`` that represents the throughput of the
1115  atmosphere, assumed to be spatially constant.
1116  detectorNum : `int`, optional
1117  The integer number for the detector to process.
1118  isGen3 : bool, optional
1119  Flag this call to run() as using the Gen3 butler environment.
1120  strayLightData : `object`, optional
1121  Opaque object containing calibration information for stray-light
1122  correction. If `None`, no correction will be performed.
1123  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1124  Illumination correction image.
1125 
1126  Returns
1127  -------
1128  result : `lsst.pipe.base.Struct`
1129  Result struct with component:
1130  - ``exposure`` : `afw.image.Exposure`
1131  The fully ISR corrected exposure.
1132  - ``outputExposure`` : `afw.image.Exposure`
1133  An alias for `exposure`
1134  - ``ossThumb`` : `numpy.ndarray`
1135  Thumbnail image of the exposure after overscan subtraction.
1136  - ``flattenedThumb`` : `numpy.ndarray`
1137  Thumbnail image of the exposure after flat-field correction.
1138 
1139  Raises
1140  ------
1141  RuntimeError
1142  Raised if a configuration option is set to True, but the
1143  required calibration data has not been specified.
1144 
1145  Notes
1146  -----
1147  The current processed exposure can be viewed by setting the
1148  appropriate lsstDebug entries in the `debug.display`
1149  dictionary. The names of these entries correspond to some of
1150  the IsrTaskConfig Boolean options, with the value denoting the
1151  frame to use. The exposure is shown inside the matching
1152  option check and after the processing of that step has
1153  finished. The steps with debug points are:
1154 
1155  doAssembleCcd
1156  doBias
1157  doCrosstalk
1158  doBrighterFatter
1159  doDark
1160  doFringe
1161  doStrayLight
1162  doFlat
1163 
1164  In addition, setting the "postISRCCD" entry displays the
1165  exposure after all ISR processing has finished.
1166 
1167  """
1168 
1169  if isGen3 is True:
1170  # Gen3 currently cannot automatically do configuration overrides.
1171  # DM-15257 looks to discuss this issue.
1172  # Configure input exposures;
1173  if detectorNum is None:
1174  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1175 
1176  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1177  bias = self.ensureExposure(bias, camera, detectorNum)
1178  dark = self.ensureExposure(dark, camera, detectorNum)
1179  flat = self.ensureExposure(flat, camera, detectorNum)
1180  else:
1181  if isinstance(ccdExposure, ButlerDataRef):
1182  return self.runDataRef(ccdExposure)
1183 
1184  ccd = ccdExposure.getDetector()
1185  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1186 
1187  if not ccd:
1188  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1189  ccd = [FakeAmp(ccdExposure, self.config)]
1190 
1191  # Validate Input
1192  if self.config.doBias and bias is None:
1193  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1194  if self.doLinearize(ccd) and linearizer is None:
1195  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1196  if self.config.doBrighterFatter and bfKernel is None:
1197  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1198  if self.config.doDark and dark is None:
1199  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1200  if self.config.doFlat and flat is None:
1201  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1202  if self.config.doDefect and defects is None:
1203  raise RuntimeError("Must supply defects if config.doDefect=True.")
1204  if (self.config.doFringe and filterName in self.fringe.config.filters and
1205  fringes.fringes is None):
1206  # The `fringes` object needs to be a pipeBase.Struct, as
1207  # we use it as a `dict` for the parameters of
1208  # `FringeTask.run()`. The `fringes.fringes` `list` may
1209  # not be `None` if `doFringe=True`. Otherwise, raise.
1210  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1211  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1212  illumMaskedImage is None):
1213  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1214 
1215  # Begin ISR processing.
1216  if self.config.doConvertIntToFloat:
1217  self.log.info("Converting exposure to floating point values.")
1218  ccdExposure = self.convertIntToFloat(ccdExposure)
1219 
1220  # Amplifier level processing.
1221  overscans = []
1222  for amp in ccd:
1223  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1224  if ccdExposure.getBBox().contains(amp.getBBox()):
1225  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1226  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1227 
1228  if self.config.doOverscan and not badAmp:
1229  # Overscan correction on amp-by-amp basis.
1230  overscanResults = self.overscanCorrection(ccdExposure, amp)
1231  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1232  if overscanResults is not None and \
1233  self.config.qa is not None and self.config.qa.saveStats is True:
1234  if isinstance(overscanResults.overscanFit, float):
1235  qaMedian = overscanResults.overscanFit
1236  qaStdev = float("NaN")
1237  else:
1238  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1239  afwMath.MEDIAN | afwMath.STDEVCLIP)
1240  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1241  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1242 
1243  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1244  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1245  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1246  amp.getName(), qaMedian, qaStdev)
1247  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1248  else:
1249  if badAmp:
1250  self.log.warn("Amplifier %s is bad.", amp.getName())
1251  overscanResults = None
1252 
1253  overscans.append(overscanResults if overscanResults is not None else None)
1254  else:
1255  self.log.info("Skipped OSCAN for %s.", amp.getName())
1256 
1257  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1258  self.log.info("Applying crosstalk correction.")
1259  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources)
1260  self.debugView(ccdExposure, "doCrosstalk")
1261 
1262  if self.config.doAssembleCcd:
1263  self.log.info("Assembling CCD from amplifiers.")
1264  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1265 
1266  if self.config.expectWcs and not ccdExposure.getWcs():
1267  self.log.warn("No WCS found in input exposure.")
1268  self.debugView(ccdExposure, "doAssembleCcd")
1269 
1270  ossThumb = None
1271  if self.config.qa.doThumbnailOss:
1272  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1273 
1274  if self.config.doBias:
1275  self.log.info("Applying bias correction.")
1276  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1277  trimToFit=self.config.doTrimToMatchCalib)
1278  self.debugView(ccdExposure, "doBias")
1279 
1280  if self.config.doVariance:
1281  for amp, overscanResults in zip(ccd, overscans):
1282  if ccdExposure.getBBox().contains(amp.getBBox()):
1283  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1284  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1285  if overscanResults is not None:
1286  self.updateVariance(ampExposure, amp,
1287  overscanImage=overscanResults.overscanImage)
1288  else:
1289  self.updateVariance(ampExposure, amp,
1290  overscanImage=None)
1291  if self.config.qa is not None and self.config.qa.saveStats is True:
1292  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1293  afwMath.MEDIAN | afwMath.STDEVCLIP)
1294  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1295  qaStats.getValue(afwMath.MEDIAN))
1296  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1297  qaStats.getValue(afwMath.STDEVCLIP))
1298  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1299  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1300  qaStats.getValue(afwMath.STDEVCLIP))
1301 
1302  if self.doLinearize(ccd):
1303  self.log.info("Applying linearizer.")
1304  linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
1305 
1306  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1307  self.log.info("Applying crosstalk correction.")
1308  self.crosstalk.run(ccdExposure, crosstalkSources=crosstalkSources, isTrimmed=True)
1309  self.debugView(ccdExposure, "doCrosstalk")
1310 
1311  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1312  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1313  if self.config.doDefect:
1314  self.log.info("Masking defects.")
1315  self.maskDefect(ccdExposure, defects)
1316 
1317  if self.config.numEdgeSuspect > 0:
1318  self.log.info("Masking edges as SUSPECT.")
1319  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1320  maskPlane="SUSPECT")
1321 
1322  if self.config.doNanMasking:
1323  self.log.info("Masking NAN value pixels.")
1324  self.maskNan(ccdExposure)
1325 
1326  if self.config.doWidenSaturationTrails:
1327  self.log.info("Widening saturation trails.")
1328  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1329 
1330  if self.config.doCameraSpecificMasking:
1331  self.log.info("Masking regions for camera specific reasons.")
1332  self.masking.run(ccdExposure)
1333 
1334  if self.config.doBrighterFatter:
1335  # We need to apply flats and darks before we can interpolate, and we
1336  # need to interpolate before we do B-F, but we do B-F without the
1337  # flats and darks applied so we can work in units of electrons or holes.
1338  # This context manager applies and then removes the darks and flats.
1339  #
1340  # We also do not want to interpolate values here, so operate on temporary
1341  # images so we can apply only the BF-correction and roll back the
1342  # interpolation.
1343  interpExp = ccdExposure.clone()
1344  with self.flatContext(interpExp, flat, dark):
1345  isrFunctions.interpolateFromMask(
1346  maskedImage=interpExp.getMaskedImage(),
1347  fwhm=self.config.fwhm,
1348  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1349  maskNameList=self.config.maskListToInterpolate
1350  )
1351  bfExp = interpExp.clone()
1352 
1353  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1354  type(bfKernel), type(bfGains))
1355  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1356  self.config.brighterFatterMaxIter,
1357  self.config.brighterFatterThreshold,
1358  self.config.brighterFatterApplyGain,
1359  bfGains)
1360  if bfResults[1] == self.config.brighterFatterMaxIter:
1361  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1362  bfResults[0])
1363  else:
1364  self.log.info("Finished brighter fatter correction in %d iterations.",
1365  bfResults[1])
1366  image = ccdExposure.getMaskedImage().getImage()
1367  bfCorr = bfExp.getMaskedImage().getImage()
1368  bfCorr -= interpExp.getMaskedImage().getImage()
1369  image += bfCorr
1370 
1371  # Applying the brighter-fatter correction applies a
1372  # convolution to the science image. At the edges this
1373  # convolution may not have sufficient valid pixels to
1374  # produce a valid correction. Mark pixels within the size
1375  # of the brighter-fatter kernel as EDGE to warn of this
1376  # fact.
1377  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1378  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1379  maskPlane="EDGE")
1380 
1381  if self.config.brighterFatterMaskGrowSize > 0:
1382  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1383  for maskPlane in self.config.maskListToInterpolate:
1384  isrFunctions.growMasks(ccdExposure.getMask(),
1385  radius=self.config.brighterFatterMaskGrowSize,
1386  maskNameList=maskPlane,
1387  maskValue=maskPlane)
1388 
1389  self.debugView(ccdExposure, "doBrighterFatter")
1390 
1391  if self.config.doDark:
1392  self.log.info("Applying dark correction.")
1393  self.darkCorrection(ccdExposure, dark)
1394  self.debugView(ccdExposure, "doDark")
1395 
1396  if self.config.doFringe and not self.config.fringeAfterFlat:
1397  self.log.info("Applying fringe correction before flat.")
1398  self.fringe.run(ccdExposure, **fringes.getDict())
1399  self.debugView(ccdExposure, "doFringe")
1400 
1401  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1402  self.log.info("Checking strayLight correction.")
1403  self.strayLight.run(ccdExposure, strayLightData)
1404  self.debugView(ccdExposure, "doStrayLight")
1405 
1406  if self.config.doFlat:
1407  self.log.info("Applying flat correction.")
1408  self.flatCorrection(ccdExposure, flat)
1409  self.debugView(ccdExposure, "doFlat")
1410 
1411  if self.config.doApplyGains:
1412  self.log.info("Applying gain correction instead of flat.")
1413  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1414 
1415  if self.config.doFringe and self.config.fringeAfterFlat:
1416  self.log.info("Applying fringe correction after flat.")
1417  self.fringe.run(ccdExposure, **fringes.getDict())
1418 
1419  if self.config.doVignette:
1420  self.log.info("Constructing Vignette polygon.")
1421  self.vignettePolygon = self.vignette.run(ccdExposure)
1422 
1423  if self.config.vignette.doWriteVignettePolygon:
1424  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1425 
1426  if self.config.doAttachTransmissionCurve:
1427  self.log.info("Adding transmission curves.")
1428  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1429  filterTransmission=filterTransmission,
1430  sensorTransmission=sensorTransmission,
1431  atmosphereTransmission=atmosphereTransmission)
1432 
1433  flattenedThumb = None
1434  if self.config.qa.doThumbnailFlattened:
1435  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1436 
1437  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1438  self.log.info("Performing illumination correction.")
1439  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1440  illumMaskedImage, illumScale=self.config.illumScale,
1441  trimToFit=self.config.doTrimToMatchCalib)
1442 
1443  preInterpExp = None
1444  if self.config.doSaveInterpPixels:
1445  preInterpExp = ccdExposure.clone()
1446 
1447  # Reset and interpolate bad pixels.
1448  #
1449  # Large contiguous bad regions (which should have the BAD mask
1450  # bit set) should have their values set to the image median.
1451  # This group should include defects and bad amplifiers. As the
1452  # area covered by these defects are large, there's little
1453  # reason to expect that interpolation would provide a more
1454  # useful value.
1455  #
1456  # Smaller defects can be safely interpolated after the larger
1457  # regions have had their pixel values reset. This ensures
1458  # that the remaining defects adjacent to bad amplifiers (as an
1459  # example) do not attempt to interpolate extreme values.
1460  if self.config.doSetBadRegions:
1461  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1462  if badPixelCount > 0:
1463  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1464 
1465  if self.config.doInterpolate:
1466  self.log.info("Interpolating masked pixels.")
1467  isrFunctions.interpolateFromMask(
1468  maskedImage=ccdExposure.getMaskedImage(),
1469  fwhm=self.config.fwhm,
1470  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1471  maskNameList=list(self.config.maskListToInterpolate)
1472  )
1473 
1474  self.roughZeroPoint(ccdExposure)
1475 
1476  if self.config.doMeasureBackground:
1477  self.log.info("Measuring background level.")
1478  self.measureBackground(ccdExposure, self.config.qa)
1479 
1480  if self.config.qa is not None and self.config.qa.saveStats is True:
1481  for amp in ccd:
1482  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1483  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1484  afwMath.MEDIAN | afwMath.STDEVCLIP)
1485  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1486  qaStats.getValue(afwMath.MEDIAN))
1487  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1488  qaStats.getValue(afwMath.STDEVCLIP))
1489  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1490  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1491  qaStats.getValue(afwMath.STDEVCLIP))
1492 
1493  self.debugView(ccdExposure, "postISRCCD")
1494 
1495  return pipeBase.Struct(
1496  exposure=ccdExposure,
1497  ossThumb=ossThumb,
1498  flattenedThumb=flattenedThumb,
1499 
1500  preInterpolatedExposure=preInterpExp,
1501  outputExposure=ccdExposure,
1502  outputOssThumbnail=ossThumb,
1503  outputFlattenedThumbnail=flattenedThumb,
1504  )
1505 
1506  @pipeBase.timeMethod
1507  def runDataRef(self, sensorRef):
1508  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1509 
1510  This method contains the `CmdLineTask` interface to the ISR
1511  processing. All IO is handled here, freeing the `run()` method
1512  to manage only pixel-level calculations. The steps performed
1513  are:
1514  - Read in necessary detrending/isr/calibration data.
1515  - Process raw exposure in `run()`.
1516  - Persist the ISR-corrected exposure as "postISRCCD" if
1517  config.doWrite=True.
1518 
1519  Parameters
1520  ----------
1521  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1522  DataRef of the detector data to be processed
1523 
1524  Returns
1525  -------
1526  result : `lsst.pipe.base.Struct`
1527  Result struct with component:
1528  - ``exposure`` : `afw.image.Exposure`
1529  The fully ISR corrected exposure.
1530 
1531  Raises
1532  ------
1533  RuntimeError
1534  Raised if a configuration option is set to True, but the
1535  required calibration data does not exist.
1536 
1537  """
1538  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1539 
1540  ccdExposure = sensorRef.get(self.config.datasetType)
1541 
1542  camera = sensorRef.get("camera")
1543  isrData = self.readIsrData(sensorRef, ccdExposure)
1544 
1545  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1546 
1547  if self.config.doWrite:
1548  sensorRef.put(result.exposure, "postISRCCD")
1549  if result.preInterpolatedExposure is not None:
1550  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1551  if result.ossThumb is not None:
1552  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1553  if result.flattenedThumb is not None:
1554  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1555 
1556  return result
1557 
1558  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1559  """!Retrieve a calibration dataset for removing instrument signature.
1560 
1561  Parameters
1562  ----------
1563 
1564  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1565  DataRef of the detector data to find calibration datasets
1566  for.
1567  datasetType : `str`
1568  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1569  dateObs : `str`, optional
1570  Date of the observation. Used to correct butler failures
1571  when using fallback filters.
1572  immediate : `Bool`
1573  If True, disable butler proxies to enable error handling
1574  within this routine.
1575 
1576  Returns
1577  -------
1578  exposure : `lsst.afw.image.Exposure`
1579  Requested calibration frame.
1580 
1581  Raises
1582  ------
1583  RuntimeError
1584  Raised if no matching calibration frame can be found.
1585  """
1586  try:
1587  exp = dataRef.get(datasetType, immediate=immediate)
1588  except Exception as exc1:
1589  if not self.config.fallbackFilterName:
1590  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1591  try:
1592  if self.config.useFallbackDate and dateObs:
1593  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1594  dateObs=dateObs, immediate=immediate)
1595  else:
1596  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1597  except Exception as exc2:
1598  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1599  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1600  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1601 
1602  if self.config.doAssembleIsrExposures:
1603  exp = self.assembleCcd.assembleCcd(exp)
1604  return exp
1605 
1606  def ensureExposure(self, inputExp, camera, detectorNum):
1607  """Ensure that the data returned by Butler is a fully constructed exposure.
1608 
1609  ISR requires exposure-level image data for historical reasons, so if we did
1610  not recieve that from Butler, construct it from what we have, modifying the
1611  input in place.
1612 
1613  Parameters
1614  ----------
1615  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1616  `lsst.afw.image.ImageF`
1617  The input data structure obtained from Butler.
1618  camera : `lsst.afw.cameraGeom.camera`
1619  The camera associated with the image. Used to find the appropriate
1620  detector.
1621  detectorNum : `int`
1622  The detector this exposure should match.
1623 
1624  Returns
1625  -------
1626  inputExp : `lsst.afw.image.Exposure`
1627  The re-constructed exposure, with appropriate detector parameters.
1628 
1629  Raises
1630  ------
1631  TypeError
1632  Raised if the input data cannot be used to construct an exposure.
1633  """
1634  if isinstance(inputExp, afwImage.DecoratedImageU):
1635  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1636  elif isinstance(inputExp, afwImage.ImageF):
1637  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1638  elif isinstance(inputExp, afwImage.MaskedImageF):
1639  inputExp = afwImage.makeExposure(inputExp)
1640  elif isinstance(inputExp, afwImage.Exposure):
1641  pass
1642  elif inputExp is None:
1643  # Assume this will be caught by the setup if it is a problem.
1644  return inputExp
1645  else:
1646  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1647  (type(inputExp), ))
1648 
1649  if inputExp.getDetector() is None:
1650  inputExp.setDetector(camera[detectorNum])
1651 
1652  return inputExp
1653 
1654  def convertIntToFloat(self, exposure):
1655  """Convert exposure image from uint16 to float.
1656 
1657  If the exposure does not need to be converted, the input is
1658  immediately returned. For exposures that are converted to use
1659  floating point pixels, the variance is set to unity and the
1660  mask to zero.
1661 
1662  Parameters
1663  ----------
1664  exposure : `lsst.afw.image.Exposure`
1665  The raw exposure to be converted.
1666 
1667  Returns
1668  -------
1669  newexposure : `lsst.afw.image.Exposure`
1670  The input ``exposure``, converted to floating point pixels.
1671 
1672  Raises
1673  ------
1674  RuntimeError
1675  Raised if the exposure type cannot be converted to float.
1676 
1677  """
1678  if isinstance(exposure, afwImage.ExposureF):
1679  # Nothing to be done
1680  self.log.debug("Exposure already of type float.")
1681  return exposure
1682  if not hasattr(exposure, "convertF"):
1683  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1684 
1685  newexposure = exposure.convertF()
1686  newexposure.variance[:] = 1
1687  newexposure.mask[:] = 0x0
1688 
1689  return newexposure
1690 
1691  def maskAmplifier(self, ccdExposure, amp, defects):
1692  """Identify bad amplifiers, saturated and suspect pixels.
1693 
1694  Parameters
1695  ----------
1696  ccdExposure : `lsst.afw.image.Exposure`
1697  Input exposure to be masked.
1698  amp : `lsst.afw.table.AmpInfoCatalog`
1699  Catalog of parameters defining the amplifier on this
1700  exposure to mask.
1701  defects : `lsst.meas.algorithms.Defects`
1702  List of defects. Used to determine if the entire
1703  amplifier is bad.
1704 
1705  Returns
1706  -------
1707  badAmp : `Bool`
1708  If this is true, the entire amplifier area is covered by
1709  defects and unusable.
1710 
1711  """
1712  maskedImage = ccdExposure.getMaskedImage()
1713 
1714  badAmp = False
1715 
1716  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1717  # comparison with current defects definition.
1718  if defects is not None:
1719  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1720 
1721  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1722  # association with pixels in current ccdExposure).
1723  if badAmp:
1724  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1725  afwImage.PARENT)
1726  maskView = dataView.getMask()
1727  maskView |= maskView.getPlaneBitMask("BAD")
1728  del maskView
1729  return badAmp
1730 
1731  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1732  # Saturation and suspect pixels can be masked now, though.
1733  limits = dict()
1734  if self.config.doSaturation and not badAmp:
1735  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1736  if self.config.doSuspect and not badAmp:
1737  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1738  if math.isfinite(self.config.saturation):
1739  limits.update({self.config.saturatedMaskName: self.config.saturation})
1740 
1741  for maskName, maskThreshold in limits.items():
1742  if not math.isnan(maskThreshold):
1743  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1744  isrFunctions.makeThresholdMask(
1745  maskedImage=dataView,
1746  threshold=maskThreshold,
1747  growFootprints=0,
1748  maskName=maskName
1749  )
1750 
1751  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1752  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1753  afwImage.PARENT)
1754  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1755  self.config.suspectMaskName])
1756  if numpy.all(maskView.getArray() & maskVal > 0):
1757  badAmp = True
1758  maskView |= maskView.getPlaneBitMask("BAD")
1759 
1760  return badAmp
1761 
1762  def overscanCorrection(self, ccdExposure, amp):
1763  """Apply overscan correction in place.
1764 
1765  This method does initial pixel rejection of the overscan
1766  region. The overscan can also be optionally segmented to
1767  allow for discontinuous overscan responses to be fit
1768  separately. The actual overscan subtraction is performed by
1769  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1770  which is called here after the amplifier is preprocessed.
1771 
1772  Parameters
1773  ----------
1774  ccdExposure : `lsst.afw.image.Exposure`
1775  Exposure to have overscan correction performed.
1776  amp : `lsst.afw.table.AmpInfoCatalog`
1777  The amplifier to consider while correcting the overscan.
1778 
1779  Returns
1780  -------
1781  overscanResults : `lsst.pipe.base.Struct`
1782  Result struct with components:
1783  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1784  Value or fit subtracted from the amplifier image data.
1785  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1786  Value or fit subtracted from the overscan image data.
1787  - ``overscanImage`` : `lsst.afw.image.Image`
1788  Image of the overscan region with the overscan
1789  correction applied. This quantity is used to estimate
1790  the amplifier read noise empirically.
1791 
1792  Raises
1793  ------
1794  RuntimeError
1795  Raised if the ``amp`` does not contain raw pixel information.
1796 
1797  See Also
1798  --------
1799  lsst.ip.isr.isrFunctions.overscanCorrection
1800  """
1801  if not amp.getHasRawInfo():
1802  raise RuntimeError("This method must be executed on an amp with raw information.")
1803 
1804  if amp.getRawHorizontalOverscanBBox().isEmpty():
1805  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1806  return None
1807 
1808  statControl = afwMath.StatisticsControl()
1809  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1810 
1811  # Determine the bounding boxes
1812  dataBBox = amp.getRawDataBBox()
1813  oscanBBox = amp.getRawHorizontalOverscanBBox()
1814  dx0 = 0
1815  dx1 = 0
1816 
1817  prescanBBox = amp.getRawPrescanBBox()
1818  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1819  dx0 += self.config.overscanNumLeadingColumnsToSkip
1820  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1821  else:
1822  dx0 += self.config.overscanNumTrailingColumnsToSkip
1823  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1824 
1825  # Determine if we need to work on subregions of the amplifier and overscan.
1826  imageBBoxes = []
1827  overscanBBoxes = []
1828 
1829  if ((self.config.overscanBiasJump and
1830  self.config.overscanBiasJumpLocation) and
1831  (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1832  ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1833  self.config.overscanBiasJumpDevices)):
1834  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1835  yLower = self.config.overscanBiasJumpLocation
1836  yUpper = dataBBox.getHeight() - yLower
1837  else:
1838  yUpper = self.config.overscanBiasJumpLocation
1839  yLower = dataBBox.getHeight() - yUpper
1840 
1841  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1842  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1843  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1844  lsst.geom.Extent2I(dx0, 0),
1845  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1846  yLower)))
1847 
1848  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1849  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1850  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1851  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1852  yUpper)))
1853  else:
1854  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1855  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1856  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1857  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1858  oscanBBox.getHeight())))
1859 
1860  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1861  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1862  ampImage = ccdExposure.maskedImage[imageBBox]
1863  overscanImage = ccdExposure.maskedImage[overscanBBox]
1864 
1865  overscanArray = overscanImage.image.array
1866  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1867  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1868  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1869 
1870  statControl = afwMath.StatisticsControl()
1871  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1872 
1873  overscanResults = isrFunctions.overscanCorrection(ampMaskedImage=ampImage,
1874  overscanImage=overscanImage,
1875  fitType=self.config.overscanFitType,
1876  order=self.config.overscanOrder,
1877  collapseRej=self.config.overscanNumSigmaClip,
1878  statControl=statControl,
1879  overscanIsInt=self.config.overscanIsInt
1880  )
1881 
1882  # Measure average overscan levels and record them in the metadata.
1883  levelStat = afwMath.MEDIAN
1884  sigmaStat = afwMath.STDEVCLIP
1885 
1886  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1887  self.config.qa.flatness.nIter)
1888  metadata = ccdExposure.getMetadata()
1889  ampNum = amp.getName()
1890  if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1891  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1892  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1893  else:
1894  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1895  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1896  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1897 
1898  return overscanResults
1899 
1900  def updateVariance(self, ampExposure, amp, overscanImage=None):
1901  """Set the variance plane using the amplifier gain and read noise
1902 
1903  The read noise is calculated from the ``overscanImage`` if the
1904  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1905  the value from the amplifier data is used.
1906 
1907  Parameters
1908  ----------
1909  ampExposure : `lsst.afw.image.Exposure`
1910  Exposure to process.
1911  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1912  Amplifier detector data.
1913  overscanImage : `lsst.afw.image.MaskedImage`, optional.
1914  Image of overscan, required only for empirical read noise.
1915 
1916  See also
1917  --------
1918  lsst.ip.isr.isrFunctions.updateVariance
1919  """
1920  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1921  gain = amp.getGain()
1922 
1923  if math.isnan(gain):
1924  gain = 1.0
1925  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1926  elif gain <= 0:
1927  patchedGain = 1.0
1928  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
1929  amp.getName(), gain, patchedGain)
1930  gain = patchedGain
1931 
1932  if self.config.doEmpiricalReadNoise and overscanImage is None:
1933  self.log.info("Overscan is none for EmpiricalReadNoise.")
1934 
1935  if self.config.doEmpiricalReadNoise and overscanImage is not None:
1936  stats = afwMath.StatisticsControl()
1937  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1938  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
1939  self.log.info("Calculated empirical read noise for amp %s: %f.",
1940  amp.getName(), readNoise)
1941  else:
1942  readNoise = amp.getReadNoise()
1943 
1944  isrFunctions.updateVariance(
1945  maskedImage=ampExposure.getMaskedImage(),
1946  gain=gain,
1947  readNoise=readNoise,
1948  )
1949 
1950  def darkCorrection(self, exposure, darkExposure, invert=False):
1951  """!Apply dark correction in place.
1952 
1953  Parameters
1954  ----------
1955  exposure : `lsst.afw.image.Exposure`
1956  Exposure to process.
1957  darkExposure : `lsst.afw.image.Exposure`
1958  Dark exposure of the same size as ``exposure``.
1959  invert : `Bool`, optional
1960  If True, re-add the dark to an already corrected image.
1961 
1962  Raises
1963  ------
1964  RuntimeError
1965  Raised if either ``exposure`` or ``darkExposure`` do not
1966  have their dark time defined.
1967 
1968  See Also
1969  --------
1970  lsst.ip.isr.isrFunctions.darkCorrection
1971  """
1972  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1973  if math.isnan(expScale):
1974  raise RuntimeError("Exposure darktime is NAN.")
1975  if darkExposure.getInfo().getVisitInfo() is not None:
1976  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1977  else:
1978  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1979  # so getDarkTime() does not exist.
1980  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1981  darkScale = 1.0
1982 
1983  if math.isnan(darkScale):
1984  raise RuntimeError("Dark calib darktime is NAN.")
1985  isrFunctions.darkCorrection(
1986  maskedImage=exposure.getMaskedImage(),
1987  darkMaskedImage=darkExposure.getMaskedImage(),
1988  expScale=expScale,
1989  darkScale=darkScale,
1990  invert=invert,
1991  trimToFit=self.config.doTrimToMatchCalib
1992  )
1993 
1994  def doLinearize(self, detector):
1995  """!Check if linearization is needed for the detector cameraGeom.
1996 
1997  Checks config.doLinearize and the linearity type of the first
1998  amplifier.
1999 
2000  Parameters
2001  ----------
2002  detector : `lsst.afw.cameraGeom.Detector`
2003  Detector to get linearity type from.
2004 
2005  Returns
2006  -------
2007  doLinearize : `Bool`
2008  If True, linearization should be performed.
2009  """
2010  return self.config.doLinearize and \
2011  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2012 
2013  def flatCorrection(self, exposure, flatExposure, invert=False):
2014  """!Apply flat correction in place.
2015 
2016  Parameters
2017  ----------
2018  exposure : `lsst.afw.image.Exposure`
2019  Exposure to process.
2020  flatExposure : `lsst.afw.image.Exposure`
2021  Flat exposure of the same size as ``exposure``.
2022  invert : `Bool`, optional
2023  If True, unflatten an already flattened image.
2024 
2025  See Also
2026  --------
2027  lsst.ip.isr.isrFunctions.flatCorrection
2028  """
2029  isrFunctions.flatCorrection(
2030  maskedImage=exposure.getMaskedImage(),
2031  flatMaskedImage=flatExposure.getMaskedImage(),
2032  scalingType=self.config.flatScalingType,
2033  userScale=self.config.flatUserScale,
2034  invert=invert,
2035  trimToFit=self.config.doTrimToMatchCalib
2036  )
2037 
2038  def saturationDetection(self, exposure, amp):
2039  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2040 
2041  Parameters
2042  ----------
2043  exposure : `lsst.afw.image.Exposure`
2044  Exposure to process. Only the amplifier DataSec is processed.
2045  amp : `lsst.afw.table.AmpInfoCatalog`
2046  Amplifier detector data.
2047 
2048  See Also
2049  --------
2050  lsst.ip.isr.isrFunctions.makeThresholdMask
2051  """
2052  if not math.isnan(amp.getSaturation()):
2053  maskedImage = exposure.getMaskedImage()
2054  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2055  isrFunctions.makeThresholdMask(
2056  maskedImage=dataView,
2057  threshold=amp.getSaturation(),
2058  growFootprints=0,
2059  maskName=self.config.saturatedMaskName,
2060  )
2061 
2062  def saturationInterpolation(self, exposure):
2063  """!Interpolate over saturated pixels, in place.
2064 
2065  This method should be called after `saturationDetection`, to
2066  ensure that the saturated pixels have been identified in the
2067  SAT mask. It should also be called after `assembleCcd`, since
2068  saturated regions may cross amplifier boundaries.
2069 
2070  Parameters
2071  ----------
2072  exposure : `lsst.afw.image.Exposure`
2073  Exposure to process.
2074 
2075  See Also
2076  --------
2077  lsst.ip.isr.isrTask.saturationDetection
2078  lsst.ip.isr.isrFunctions.interpolateFromMask
2079  """
2080  isrFunctions.interpolateFromMask(
2081  maskedImage=exposure.getMaskedImage(),
2082  fwhm=self.config.fwhm,
2083  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2084  maskNameList=list(self.config.saturatedMaskName),
2085  )
2086 
2087  def suspectDetection(self, exposure, amp):
2088  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2089 
2090  Parameters
2091  ----------
2092  exposure : `lsst.afw.image.Exposure`
2093  Exposure to process. Only the amplifier DataSec is processed.
2094  amp : `lsst.afw.table.AmpInfoCatalog`
2095  Amplifier detector data.
2096 
2097  See Also
2098  --------
2099  lsst.ip.isr.isrFunctions.makeThresholdMask
2100 
2101  Notes
2102  -----
2103  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2104  This is intended to indicate pixels that may be affected by unknown systematics;
2105  for example if non-linearity corrections above a certain level are unstable
2106  then that would be a useful value for suspectLevel. A value of `nan` indicates
2107  that no such level exists and no pixels are to be masked as suspicious.
2108  """
2109  suspectLevel = amp.getSuspectLevel()
2110  if math.isnan(suspectLevel):
2111  return
2112 
2113  maskedImage = exposure.getMaskedImage()
2114  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2115  isrFunctions.makeThresholdMask(
2116  maskedImage=dataView,
2117  threshold=suspectLevel,
2118  growFootprints=0,
2119  maskName=self.config.suspectMaskName,
2120  )
2121 
2122  def maskDefect(self, exposure, defectBaseList):
2123  """!Mask defects using mask plane "BAD", in place.
2124 
2125  Parameters
2126  ----------
2127  exposure : `lsst.afw.image.Exposure`
2128  Exposure to process.
2129  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2130  `lsst.afw.image.DefectBase`.
2131  List of defects to mask.
2132 
2133  Notes
2134  -----
2135  Call this after CCD assembly, since defects may cross amplifier boundaries.
2136  """
2137  maskedImage = exposure.getMaskedImage()
2138  if not isinstance(defectBaseList, Defects):
2139  # Promotes DefectBase to Defect
2140  defectList = Defects(defectBaseList)
2141  else:
2142  defectList = defectBaseList
2143  defectList.maskPixels(maskedImage, maskName="BAD")
2144 
2145  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2146  """!Mask edge pixels with applicable mask plane.
2147 
2148  Parameters
2149  ----------
2150  exposure : `lsst.afw.image.Exposure`
2151  Exposure to process.
2152  numEdgePixels : `int`, optional
2153  Number of edge pixels to mask.
2154  maskPlane : `str`, optional
2155  Mask plane name to use.
2156  """
2157  maskedImage = exposure.getMaskedImage()
2158  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2159 
2160  if numEdgePixels > 0:
2161  goodBBox = maskedImage.getBBox()
2162  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2163  goodBBox.grow(-numEdgePixels)
2164  # Mask pixels outside goodBBox
2165  SourceDetectionTask.setEdgeBits(
2166  maskedImage,
2167  goodBBox,
2168  maskBitMask
2169  )
2170 
2171  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2172  """Mask and interpolate defects using mask plane "BAD", in place.
2173 
2174  Parameters
2175  ----------
2176  exposure : `lsst.afw.image.Exposure`
2177  Exposure to process.
2178  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2179  `lsst.afw.image.DefectBase`.
2180  List of defects to mask and interpolate.
2181 
2182  See Also
2183  --------
2184  lsst.ip.isr.isrTask.maskDefect()
2185  """
2186  self.maskDefect(exposure, defectBaseList)
2187  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2188  maskPlane="SUSPECT")
2189  isrFunctions.interpolateFromMask(
2190  maskedImage=exposure.getMaskedImage(),
2191  fwhm=self.config.fwhm,
2192  growSaturatedFootprints=0,
2193  maskNameList=["BAD"],
2194  )
2195 
2196  def maskNan(self, exposure):
2197  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2198 
2199  Parameters
2200  ----------
2201  exposure : `lsst.afw.image.Exposure`
2202  Exposure to process.
2203 
2204  Notes
2205  -----
2206  We mask over all NaNs, including those that are masked with
2207  other bits (because those may or may not be interpolated over
2208  later, and we want to remove all NaNs). Despite this
2209  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2210  the historical name.
2211  """
2212  maskedImage = exposure.getMaskedImage()
2213 
2214  # Find and mask NaNs
2215  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2216  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2217  numNans = maskNans(maskedImage, maskVal)
2218  self.metadata.set("NUMNANS", numNans)
2219  if numNans > 0:
2220  self.log.warn("There were %d unmasked NaNs.", numNans)
2221 
2222  def maskAndInterpolateNan(self, exposure):
2223  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2224 
2225  Parameters
2226  ----------
2227  exposure : `lsst.afw.image.Exposure`
2228  Exposure to process.
2229 
2230  See Also
2231  --------
2232  lsst.ip.isr.isrTask.maskNan()
2233  """
2234  self.maskNan(exposure)
2235  isrFunctions.interpolateFromMask(
2236  maskedImage=exposure.getMaskedImage(),
2237  fwhm=self.config.fwhm,
2238  growSaturatedFootprints=0,
2239  maskNameList=["UNMASKEDNAN"],
2240  )
2241 
2242  def measureBackground(self, exposure, IsrQaConfig=None):
2243  """Measure the image background in subgrids, for quality control purposes.
2244 
2245  Parameters
2246  ----------
2247  exposure : `lsst.afw.image.Exposure`
2248  Exposure to process.
2249  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2250  Configuration object containing parameters on which background
2251  statistics and subgrids to use.
2252  """
2253  if IsrQaConfig is not None:
2254  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2255  IsrQaConfig.flatness.nIter)
2256  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2257  statsControl.setAndMask(maskVal)
2258  maskedImage = exposure.getMaskedImage()
2259  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2260  skyLevel = stats.getValue(afwMath.MEDIAN)
2261  skySigma = stats.getValue(afwMath.STDEVCLIP)
2262  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2263  metadata = exposure.getMetadata()
2264  metadata.set('SKYLEVEL', skyLevel)
2265  metadata.set('SKYSIGMA', skySigma)
2266 
2267  # calcluating flatlevel over the subgrids
2268  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2269  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2270  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2271  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2272  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2273  skyLevels = numpy.zeros((nX, nY))
2274 
2275  for j in range(nY):
2276  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2277  for i in range(nX):
2278  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2279 
2280  xLLC = xc - meshXHalf
2281  yLLC = yc - meshYHalf
2282  xURC = xc + meshXHalf - 1
2283  yURC = yc + meshYHalf - 1
2284 
2285  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2286  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2287 
2288  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2289 
2290  good = numpy.where(numpy.isfinite(skyLevels))
2291  skyMedian = numpy.median(skyLevels[good])
2292  flatness = (skyLevels[good] - skyMedian) / skyMedian
2293  flatness_rms = numpy.std(flatness)
2294  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2295 
2296  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2297  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2298  nX, nY, flatness_pp, flatness_rms)
2299 
2300  metadata.set('FLATNESS_PP', float(flatness_pp))
2301  metadata.set('FLATNESS_RMS', float(flatness_rms))
2302  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2303  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2304  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2305 
2306  def roughZeroPoint(self, exposure):
2307  """Set an approximate magnitude zero point for the exposure.
2308 
2309  Parameters
2310  ----------
2311  exposure : `lsst.afw.image.Exposure`
2312  Exposure to process.
2313  """
2314  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2315  if filterName in self.config.fluxMag0T1:
2316  fluxMag0 = self.config.fluxMag0T1[filterName]
2317  else:
2318  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2319  fluxMag0 = self.config.defaultFluxMag0T1
2320 
2321  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2322  if not expTime > 0: # handle NaN as well as <= 0
2323  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2324  return
2325 
2326  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2327  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2328 
2329  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2330  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2331 
2332  Parameters
2333  ----------
2334  ccdExposure : `lsst.afw.image.Exposure`
2335  Exposure to process.
2336  fpPolygon : `lsst.afw.geom.Polygon`
2337  Polygon in focal plane coordinates.
2338  """
2339  # Get ccd corners in focal plane coordinates
2340  ccd = ccdExposure.getDetector()
2341  fpCorners = ccd.getCorners(FOCAL_PLANE)
2342  ccdPolygon = Polygon(fpCorners)
2343 
2344  # Get intersection of ccd corners with fpPolygon
2345  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2346 
2347  # Transform back to pixel positions and build new polygon
2348  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2349  validPolygon = Polygon(ccdPoints)
2350  ccdExposure.getInfo().setValidPolygon(validPolygon)
2351 
2352  @contextmanager
2353  def flatContext(self, exp, flat, dark=None):
2354  """Context manager that applies and removes flats and darks,
2355  if the task is configured to apply them.
2356 
2357  Parameters
2358  ----------
2359  exp : `lsst.afw.image.Exposure`
2360  Exposure to process.
2361  flat : `lsst.afw.image.Exposure`
2362  Flat exposure the same size as ``exp``.
2363  dark : `lsst.afw.image.Exposure`, optional
2364  Dark exposure the same size as ``exp``.
2365 
2366  Yields
2367  ------
2368  exp : `lsst.afw.image.Exposure`
2369  The flat and dark corrected exposure.
2370  """
2371  if self.config.doDark and dark is not None:
2372  self.darkCorrection(exp, dark)
2373  if self.config.doFlat:
2374  self.flatCorrection(exp, flat)
2375  try:
2376  yield exp
2377  finally:
2378  if self.config.doFlat:
2379  self.flatCorrection(exp, flat, invert=True)
2380  if self.config.doDark and dark is not None:
2381  self.darkCorrection(exp, dark, invert=True)
2382 
2383  def debugView(self, exposure, stepname):
2384  """Utility function to examine ISR exposure at different stages.
2385 
2386  Parameters
2387  ----------
2388  exposure : `lsst.afw.image.Exposure`
2389  Exposure to view.
2390  stepname : `str`
2391  State of processing to view.
2392  """
2393  frame = getDebugFrame(self._display, stepname)
2394  if frame:
2395  display = getDisplay(frame)
2396  display.scale('asinh', 'zscale')
2397  display.mtv(exposure)
2398  prompt = "Press Enter to continue [c]... "
2399  while True:
2400  ans = input(prompt).lower()
2401  if ans in ("", "c",):
2402  break
2403 
2404 
2405 class FakeAmp(object):
2406  """A Detector-like object that supports returning gain and saturation level
2407 
2408  This is used when the input exposure does not have a detector.
2409 
2410  Parameters
2411  ----------
2412  exposure : `lsst.afw.image.Exposure`
2413  Exposure to generate a fake amplifier for.
2414  config : `lsst.ip.isr.isrTaskConfig`
2415  Configuration to apply to the fake amplifier.
2416  """
2417 
2418  def __init__(self, exposure, config):
2419  self._bbox = exposure.getBBox(afwImage.LOCAL)
2421  self._gain = config.gain
2422  self._readNoise = config.readNoise
2423  self._saturation = config.saturation
2424 
2425  def getBBox(self):
2426  return self._bbox
2427 
2428  def getRawBBox(self):
2429  return self._bbox
2430 
2431  def getHasRawInfo(self):
2432  return True # but see getRawHorizontalOverscanBBox()
2433 
2435  return self._RawHorizontalOverscanBBox
2436 
2437  def getGain(self):
2438  return self._gain
2439 
2440  def getReadNoise(self):
2441  return self._readNoise
2442 
2443  def getSaturation(self):
2444  return self._saturation
2445 
2446  def getSuspectLevel(self):
2447  return float("NaN")
2448 
2449 
2450 class RunIsrConfig(pexConfig.Config):
2451  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2452 
2453 
2454 class RunIsrTask(pipeBase.CmdLineTask):
2455  """Task to wrap the default IsrTask to allow it to be retargeted.
2456 
2457  The standard IsrTask can be called directly from a command line
2458  program, but doing so removes the ability of the task to be
2459  retargeted. As most cameras override some set of the IsrTask
2460  methods, this would remove those data-specific methods in the
2461  output post-ISR images. This wrapping class fixes the issue,
2462  allowing identical post-ISR images to be generated by both the
2463  processCcd and isrTask code.
2464  """
2465  ConfigClass = RunIsrConfig
2466  _DefaultName = "runIsr"
2467 
2468  def __init__(self, *args, **kwargs):
2469  super().__init__(*args, **kwargs)
2470  self.makeSubtask("isr")
2471 
2472  def runDataRef(self, dataRef):
2473  """
2474  Parameters
2475  ----------
2476  dataRef : `lsst.daf.persistence.ButlerDataRef`
2477  data reference of the detector data to be processed
2478 
2479  Returns
2480  -------
2481  result : `pipeBase.Struct`
2482  Result struct with component:
2483 
2484  - exposure : `lsst.afw.image.Exposure`
2485  Post-ISR processed exposure.
2486  """
2487  return self.isr.runDataRef(dataRef)
def runDataRef(self, sensorRef)
Definition: isrTask.py:1507
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2242
def debugView(self, exposure, stepname)
Definition: isrTask.py:2383
def __init__(self, kwargs)
Definition: isrTask.py:808
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1606
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:886
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2145
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:817
def runDataRef(self, dataRef)
Definition: isrTask.py:2472
def __init__(self, args, kwargs)
Definition: isrTask.py:2468
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2306
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2171
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2434
def maskNan(self, exposure)
Definition: isrTask.py:2196
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2122
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1762
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1654
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:2013
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1052
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:1950
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:1994
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2329
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1691
def __init__(self, config=None)
Definition: isrTask.py:179
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2353
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1558
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1900
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2222
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2087
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2062
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place...
Definition: isrTask.py:2038
def __init__(self, exposure, config)
Definition: isrTask.py:2418