lsst.ip.isr  8.0-hsc+1
isrFunctions.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 from deprecated.sphinx import deprecated
25 
26 import lsst.geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.detection as afwDetection
29 import lsst.afw.math as afwMath
30 import lsst.meas.algorithms as measAlg
31 import lsst.pex.exceptions as pexExcept
32 import lsst.afw.cameraGeom as camGeom
33 
34 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
35 from lsst.meas.algorithms.detection import SourceDetectionTask
36 from lsst.pipe.base import Struct
37 
38 from contextlib import contextmanager
39 
40 
41 def createPsf(fwhm):
42  """Make a double Gaussian PSF.
43 
44  Parameters
45  ----------
46  fwhm : scalar
47  FWHM of double Gaussian smoothing kernel.
48 
49  Returns
50  -------
51  psf : `lsst.meas.algorithms.DoubleGaussianPsf`
52  The created smoothing kernel.
53  """
54  ksize = 4*int(fwhm) + 1
55  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
56 
57 
58 def transposeMaskedImage(maskedImage):
59  """Make a transposed copy of a masked image.
60 
61  Parameters
62  ----------
63  maskedImage : `lsst.afw.image.MaskedImage`
64  Image to process.
65 
66  Returns
67  -------
68  transposed : `lsst.afw.image.MaskedImage`
69  The transposed copy of the input image.
70  """
71  transposed = maskedImage.Factory(lsst.geom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
72  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
73  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
74  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
75  return transposed
76 
77 
78 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
79  """Interpolate over defects specified in a defect list.
80 
81  Parameters
82  ----------
83  maskedImage : `lsst.afw.image.MaskedImage`
84  Image to process.
85  defectList : `lsst.meas.algorithms.Defects`
86  List of defects to interpolate over.
87  fwhm : scalar
88  FWHM of double Gaussian smoothing kernel.
89  fallbackValue : scalar, optional
90  Fallback value if an interpolated value cannot be determined.
91  If None, then the clipped mean of the image is used.
92  """
93  psf = createPsf(fwhm)
94  if fallbackValue is None:
95  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
96  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
97  maskedImage.getMask().addMaskPlane('INTRP')
98  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
99  return maskedImage
100 
101 
102 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
103  """Mask pixels based on threshold detection.
104 
105  Parameters
106  ----------
107  maskedImage : `lsst.afw.image.MaskedImage`
108  Image to process. Only the mask plane is updated.
109  threshold : scalar
110  Detection threshold.
111  growFootprints : scalar, optional
112  Number of pixels to grow footprints of detected regions.
113  maskName : str, optional
114  Mask plane name, or list of names to convert
115 
116  Returns
117  -------
118  defectList : `lsst.meas.algorithms.Defects`
119  Defect list constructed from pixels above the threshold.
120  """
121  # find saturated regions
122  thresh = afwDetection.Threshold(threshold)
123  fs = afwDetection.FootprintSet(maskedImage, thresh)
124 
125  if growFootprints > 0:
126  fs = afwDetection.FootprintSet(fs, rGrow=growFootprints, isotropic=False)
127  fpList = fs.getFootprints()
128 
129  # set mask
130  mask = maskedImage.getMask()
131  bitmask = mask.getPlaneBitMask(maskName)
132  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
133 
134  return measAlg.Defects.fromFootprintList(fpList)
135 
136 
137 def growMasks(mask, radius=0, maskNameList=['BAD'], maskValue="BAD"):
138  """Grow a mask by an amount and add to the requested plane.
139 
140  Parameters
141  ----------
142  mask : `lsst.afw.image.Mask`
143  Mask image to process.
144  radius : scalar
145  Amount to grow the mask.
146  maskNameList : `str` or `list` [`str`]
147  Mask names that should be grown.
148  maskValue : `str`
149  Mask plane to assign the newly masked pixels to.
150  """
151  if radius > 0:
152  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK)
153  fpSet = afwDetection.FootprintSet(mask, thresh)
154  fpSet = afwDetection.FootprintSet(fpSet, rGrow=radius, isotropic=False)
155  fpSet.setMask(mask, maskValue)
156 
157 
158 def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1,
159  maskNameList=['SAT'], fallbackValue=None):
160  """Interpolate over defects identified by a particular set of mask planes.
161 
162  Parameters
163  ----------
164  maskedImage : `lsst.afw.image.MaskedImage`
165  Image to process.
166  fwhm : scalar
167  FWHM of double Gaussian smoothing kernel.
168  growSaturatedFootprints : scalar, optional
169  Number of pixels to grow footprints for saturated pixels.
170  maskNameList : `List` of `str`, optional
171  Mask plane name.
172  fallbackValue : scalar, optional
173  Value of last resort for interpolation.
174  """
175  mask = maskedImage.getMask()
176 
177  if growSaturatedFootprints > 0 and "SAT" in maskNameList:
178  # If we are interpolating over an area larger than the original masked region, we need
179  # to expand the original mask bit to the full area to explain why we interpolated there.
180  growMasks(mask, radius=growSaturatedFootprints, maskNameList=['SAT'], maskValue="SAT")
181 
182  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK)
183  fpSet = afwDetection.FootprintSet(mask, thresh)
184  defectList = measAlg.Defects.fromFootprintList(fpSet.getFootprints())
185 
186  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
187 
188  return maskedImage
189 
190 
191 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
192  fallbackValue=None):
193  """Mark saturated pixels and optionally interpolate over them
194 
195  Parameters
196  ----------
197  maskedImage : `lsst.afw.image.MaskedImage`
198  Image to process.
199  saturation : scalar
200  Saturation level used as the detection threshold.
201  fwhm : scalar
202  FWHM of double Gaussian smoothing kernel.
203  growFootprints : scalar, optional
204  Number of pixels to grow footprints of detected regions.
205  interpolate : Bool, optional
206  If True, saturated pixels are interpolated over.
207  maskName : str, optional
208  Mask plane name.
209  fallbackValue : scalar, optional
210  Value of last resort for interpolation.
211  """
212  defectList = makeThresholdMask(
213  maskedImage=maskedImage,
214  threshold=saturation,
215  growFootprints=growFootprints,
216  maskName=maskName,
217  )
218  if interpolate:
219  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
220 
221  return maskedImage
222 
223 
224 def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage):
225  """Compute number of edge trim pixels to match the calibration data.
226 
227  Use the dimension difference between the raw exposure and the
228  calibration exposure to compute the edge trim pixels. This trim
229  is applied symmetrically, with the same number of pixels masked on
230  each side.
231 
232  Parameters
233  ----------
234  rawMaskedImage : `lsst.afw.image.MaskedImage`
235  Image to trim.
236  calibMaskedImage : `lsst.afw.image.MaskedImage`
237  Calibration image to draw new bounding box from.
238 
239  Returns
240  -------
241  replacementMaskedImage : `lsst.afw.image.MaskedImage`
242  ``rawMaskedImage`` trimmed to the appropriate size
243  Raises
244  ------
245  RuntimeError
246  Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to
247  match ``calibMaskedImage``.
248  """
249  nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions()
250  if nx != ny:
251  raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.")
252  if nx % 2 != 0:
253  raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.")
254  if nx < 0:
255  raise RuntimeError("Calibration maskedImage is larger than raw data.")
256 
257  nEdge = nx//2
258  if nEdge > 0:
259  replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL]
260  SourceDetectionTask.setEdgeBits(
261  rawMaskedImage,
262  replacementMaskedImage.getBBox(),
263  rawMaskedImage.getMask().getPlaneBitMask("EDGE")
264  )
265  else:
266  replacementMaskedImage = rawMaskedImage
267 
268  return replacementMaskedImage
269 
270 
271 def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False):
272  """Apply bias correction in place.
273 
274  Parameters
275  ----------
276  maskedImage : `lsst.afw.image.MaskedImage`
277  Image to process. The image is modified by this method.
278  biasMaskedImage : `lsst.afw.image.MaskedImage`
279  Bias image of the same size as ``maskedImage``
280  trimToFit : `Bool`, optional
281  If True, raw data is symmetrically trimmed to match
282  calibration size.
283 
284  Raises
285  ------
286  RuntimeError
287  Raised if ``maskedImage`` and ``biasMaskedImage`` do not have
288  the same size.
289 
290  """
291  if trimToFit:
292  maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage)
293 
294  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
295  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
296  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
297  maskedImage -= biasMaskedImage
298 
299 
300 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False):
301  """Apply dark correction in place.
302 
303  Parameters
304  ----------
305  maskedImage : `lsst.afw.image.MaskedImage`
306  Image to process. The image is modified by this method.
307  darkMaskedImage : `lsst.afw.image.MaskedImage`
308  Dark image of the same size as ``maskedImage``.
309  expScale : scalar
310  Dark exposure time for ``maskedImage``.
311  darkScale : scalar
312  Dark exposure time for ``darkMaskedImage``.
313  invert : `Bool`, optional
314  If True, re-add the dark to an already corrected image.
315  trimToFit : `Bool`, optional
316  If True, raw data is symmetrically trimmed to match
317  calibration size.
318 
319  Raises
320  ------
321  RuntimeError
322  Raised if ``maskedImage`` and ``darkMaskedImage`` do not have
323  the same size.
324 
325  Notes
326  -----
327  The dark correction is applied by calculating:
328  maskedImage -= dark * expScaling / darkScaling
329  """
330  if trimToFit:
331  maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage)
332 
333  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
334  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
335  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
336 
337  scale = expScale / darkScale
338  if not invert:
339  maskedImage.scaledMinus(scale, darkMaskedImage)
340  else:
341  maskedImage.scaledPlus(scale, darkMaskedImage)
342 
343 
344 def updateVariance(maskedImage, gain, readNoise):
345  """Set the variance plane based on the image plane.
346 
347  Parameters
348  ----------
349  maskedImage : `lsst.afw.image.MaskedImage`
350  Image to process. The variance plane is modified.
351  gain : scalar
352  The amplifier gain in electrons/ADU.
353  readNoise : scalar
354  The amplifier read nmoise in ADU/pixel.
355  """
356  var = maskedImage.getVariance()
357  var[:] = maskedImage.getImage()
358  var /= gain
359  var += readNoise**2
360 
361 
362 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
363  """Apply flat correction in place.
364 
365  Parameters
366  ----------
367  maskedImage : `lsst.afw.image.MaskedImage`
368  Image to process. The image is modified.
369  flatMaskedImage : `lsst.afw.image.MaskedImage`
370  Flat image of the same size as ``maskedImage``
371  scalingType : str
372  Flat scale computation method. Allowed values are 'MEAN',
373  'MEDIAN', or 'USER'.
374  userScale : scalar, optional
375  Scale to use if ``scalingType``='USER'.
376  invert : `Bool`, optional
377  If True, unflatten an already flattened image.
378  trimToFit : `Bool`, optional
379  If True, raw data is symmetrically trimmed to match
380  calibration size.
381 
382  Raises
383  ------
384  RuntimeError
385  Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
386  the same size or if ``scalingType`` is not an allowed value.
387  """
388  if trimToFit:
389  maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
390 
391  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
392  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
393  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
394 
395  # Figure out scale from the data
396  # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
397  # in the case that the flat is created by some other mechanism.
398  if scalingType in ('MEAN', 'MEDIAN'):
399  scalingType = afwMath.stringToStatisticsProperty(scalingType)
400  flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
401  elif scalingType == 'USER':
402  flatScale = userScale
403  else:
404  raise RuntimeError('%s : %s not implemented' % ("flatCorrection", scalingType))
405 
406  if not invert:
407  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
408  else:
409  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
410 
411 
412 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True):
413  """Apply illumination correction in place.
414 
415  Parameters
416  ----------
417  maskedImage : `lsst.afw.image.MaskedImage`
418  Image to process. The image is modified.
419  illumMaskedImage : `lsst.afw.image.MaskedImage`
420  Illumination correction image of the same size as ``maskedImage``.
421  illumScale : scalar
422  Scale factor for the illumination correction.
423  trimToFit : `Bool`, optional
424  If True, raw data is symmetrically trimmed to match
425  calibration size.
426 
427  Raises
428  ------
429  RuntimeError
430  Raised if ``maskedImage`` and ``illumMaskedImage`` do not have
431  the same size.
432  """
433  if trimToFit:
434  maskedImage = trimToMatchCalibBBox(maskedImage, illumMaskedImage)
435 
436  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
437  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
438  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
439 
440  maskedImage.scaledDivides(1.0/illumScale, illumMaskedImage)
441 
442 
443 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
444  statControl=None, overscanIsInt=True):
445  """Apply overscan correction in place.
446 
447  Parameters
448  ----------
449  ampMaskedImage : `lsst.afw.image.MaskedImage`
450  Image of amplifier to correct; modified.
451  overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
452  Image of overscan; modified.
453  fitType : `str`
454  Type of fit for overscan correction. May be one of:
455 
456  - ``MEAN``: use mean of overscan.
457  - ``MEANCLIP``: use clipped mean of overscan.
458  - ``MEDIAN``: use median of overscan.
459  - ``POLY``: fit with ordinary polynomial.
460  - ``CHEB``: fit with Chebyshev polynomial.
461  - ``LEG``: fit with Legendre polynomial.
462  - ``NATURAL_SPLINE``: fit with natural spline.
463  - ``CUBIC_SPLINE``: fit with cubic spline.
464  - ``AKIMA_SPLINE``: fit with Akima spline.
465 
466  order : `int`
467  Polynomial order or number of spline knots; ignored unless
468  ``fitType`` indicates a polynomial or spline.
469  statControl : `lsst.afw.math.StatisticsControl`
470  Statistics control object. In particular, we pay attention to numSigmaClip
471  overscanIsInt : `bool`
472  Treat the overscan region as consisting of integers, even if it's been
473  converted to float. E.g. handle ties properly.
474 
475  Returns
476  -------
477  result : `lsst.pipe.base.Struct`
478  Result struct with components:
479 
480  - ``imageFit``: Value(s) removed from image (scalar or
481  `lsst.afw.image.Image`)
482  - ``overscanFit``: Value(s) removed from overscan (scalar or
483  `lsst.afw.image.Image`)
484  - ``overscanImage``: Overscan corrected overscan region
485  (`lsst.afw.image.Image`)
486  Raises
487  ------
488  pexExcept.Exception
489  Raised if ``fitType`` is not an allowed value.
490 
491  Notes
492  -----
493  The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
494  subtracted. Note that the ``overscanImage`` should not be a subimage of
495  the ``ampMaskedImage``, to avoid being subtracted twice.
496 
497  Debug plots are available for the SPLINE fitTypes by setting the
498  `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
499  plots show the scatter plot of the overscan data (collapsed along
500  the perpendicular dimension) as a function of position on the CCD
501  (normalized between +/-1).
502  """
503  ampImage = ampMaskedImage.getImage()
504  if statControl is None:
505  statControl = afwMath.StatisticsControl()
506 
507  numSigmaClip = statControl.getNumSigmaClip()
508 
509  if fitType in ('MEAN', 'MEANCLIP'):
510  fitType = afwMath.stringToStatisticsProperty(fitType)
511  offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
512  overscanFit = offImage
513  elif fitType in ('MEDIAN',):
514  if overscanIsInt:
515  # we need an image with integer pixels to handle ties properly
516  if hasattr(overscanImage, "image"):
517  imageI = overscanImage.image.convertI()
518  overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
519  else:
520  overscanImageI = overscanImage.convertI()
521  else:
522  overscanImageI = overscanImage
523 
524  fitType = afwMath.stringToStatisticsProperty(fitType)
525  offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
526  overscanFit = offImage
527 
528  if overscanIsInt:
529  del overscanImageI
530  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
531  if hasattr(overscanImage, "getImage"):
532  biasArray = overscanImage.getImage().getArray()
533  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
534  biasArray)
535  else:
536  biasArray = overscanImage.getArray()
537  # Fit along the long axis, so collapse along each short row and fit the resulting array
538  shortInd = numpy.argmin(biasArray.shape)
539  if shortInd == 0:
540  # Convert to some 'standard' representation to make things easier
541  biasArray = numpy.transpose(biasArray)
542 
543  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
544  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
545  medianBiasArr = percentiles[1]
546  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
547  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
548  biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
549  collapsed = numpy.mean(biasMaskedArr, axis=1)
550  if collapsed.mask.sum() > 0:
551  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
552  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
553 
554  if shortInd == 0:
555  collapsed = numpy.transpose(collapsed)
556 
557  num = len(collapsed)
558  indices = 2.0*numpy.arange(num)/float(num) - 1.0
559 
560  if fitType in ('POLY', 'CHEB', 'LEG'):
561  # A numpy polynomial
562  poly = numpy.polynomial
563  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
564  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
565  "LEG": (poly.legendre.legfit, poly.legendre.legval),
566  }[fitType]
567 
568  coeffs = fitter(indices, collapsed, order)
569  fitBiasArr = evaler(indices, coeffs)
570  elif 'SPLINE' in fitType:
571  # An afw interpolation
572  numBins = order
573  #
574  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
575  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
576  #
577  # Issue DM-415
578  #
579  collapsedMask = collapsed.mask
580  try:
581  if collapsedMask == numpy.ma.nomask:
582  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
583  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
584  pass
585 
586  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
587  weights=1-collapsedMask.astype(int))
588  # Binning is just a histogram, with weights equal to the values.
589  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
590  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
591  values = numpy.histogram(indices, bins=numBins,
592  weights=collapsed.data*~collapsedMask)[0]/numPerBin
593  binCenters = numpy.histogram(indices, bins=numBins,
594  weights=indices*~collapsedMask)[0]/numPerBin
595  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
596  values.astype(float)[numPerBin > 0],
597  afwMath.stringToInterpStyle(fitType))
598  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
599 
600  import lsstDebug
601  if lsstDebug.Info(__name__).display:
602  import matplotlib.pyplot as plot
603  figure = plot.figure(1)
604  figure.clear()
605  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
606  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
607  if collapsedMask.sum() > 0:
608  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
609  axes.plot(indices, fitBiasArr, 'r-')
610  plot.xlabel("centered/scaled position along overscan region")
611  plot.ylabel("pixel value/fit value")
612  figure.show()
613  prompt = "Press Enter or c to continue [chp]... "
614  while True:
615  ans = input(prompt).lower()
616  if ans in ("", "c",):
617  break
618  if ans in ("p",):
619  import pdb
620  pdb.set_trace()
621  elif ans in ("h", ):
622  print("h[elp] c[ontinue] p[db]")
623  plot.close()
624 
625  offImage = ampImage.Factory(ampImage.getDimensions())
626  offArray = offImage.getArray()
627  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
628  overscanArray = overscanFit.getArray()
629  if shortInd == 1:
630  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
631  overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
632  else:
633  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
634  overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
635 
636  # We don't trust any extrapolation: mask those pixels as SUSPECT
637  # This will occur when the top and or bottom edges of the overscan
638  # contain saturated values. The values will be extrapolated from
639  # the surrounding pixels, but we cannot entirely trust the value of
640  # the extrapolation, and will mark the image mask plane to flag the
641  # image as such.
642  mask = ampMaskedImage.getMask()
643  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
644  suspect = mask.getPlaneBitMask("SUSPECT")
645  try:
646  if collapsed.mask == numpy.ma.nomask:
647  # There is no mask, so the whole array is fine
648  pass
649  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
650  for low in range(num):
651  if not collapsed.mask[low]:
652  break
653  if low > 0:
654  maskArray[:low, :] |= suspect
655  for high in range(1, num):
656  if not collapsed.mask[-high]:
657  break
658  if high > 1:
659  maskArray[-high:, :] |= suspect
660 
661  else:
662  raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
663  ampImage -= offImage
664  overscanImage -= overscanFit
665  return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
666 
667 
668 def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain, gains=None):
669  """Apply brighter fatter correction in place for the image.
670 
671  Parameters
672  ----------
673  exposure : `lsst.afw.image.Exposure`
674  Exposure to have brighter-fatter correction applied. Modified
675  by this method.
676  kernel : `numpy.ndarray`
677  Brighter-fatter kernel to apply.
678  maxIter : scalar
679  Number of correction iterations to run.
680  threshold : scalar
681  Convergence threshold in terms of the sum of absolute
682  deviations between an iteration and the previous one.
683  applyGain : `Bool`
684  If True, then the exposure values are scaled by the gain prior
685  to correction.
686  gains : `dict` [`str`, `float`]
687  A dictionary, keyed by amplifier name, of the gains to use.
688  If gains is None, the nominal gains in the amplifier object are used.
689 
690  Returns
691  -------
692  diff : `float`
693  Final difference between iterations achieved in correction.
694  iteration : `int`
695  Number of iterations used to calculate correction.
696 
697  Notes
698  -----
699  This correction takes a kernel that has been derived from flat
700  field images to redistribute the charge. The gradient of the
701  kernel is the deflection field due to the accumulated charge.
702 
703  Given the original image I(x) and the kernel K(x) we can compute
704  the corrected image Ic(x) using the following equation:
705 
706  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
707 
708  To evaluate the derivative term we expand it as follows:
709 
710  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
711 
712  Because we use the measured counts instead of the incident counts
713  we apply the correction iteratively to reconstruct the original
714  counts and the correction. We stop iterating when the summed
715  difference between the current corrected image and the one from
716  the previous iteration is below the threshold. We do not require
717  convergence because the number of iterations is too large a
718  computational cost. How we define the threshold still needs to be
719  evaluated, the current default was shown to work reasonably well
720  on a small set of images. For more information on the method see
721  DocuShare Document-19407.
722 
723  The edges as defined by the kernel are not corrected because they
724  have spurious values due to the convolution.
725  """
726  image = exposure.getMaskedImage().getImage()
727 
728  # The image needs to be units of electrons/holes
729  with gainContext(exposure, image, applyGain, gains):
730 
731  kLx = numpy.shape(kernel)[0]
732  kLy = numpy.shape(kernel)[1]
733  kernelImage = afwImage.ImageD(kLx, kLy)
734  kernelImage.getArray()[:, :] = kernel
735  tempImage = image.clone()
736 
737  nanIndex = numpy.isnan(tempImage.getArray())
738  tempImage.getArray()[nanIndex] = 0.
739 
740  outImage = afwImage.ImageF(image.getDimensions())
741  corr = numpy.zeros_like(image.getArray())
742  prev_image = numpy.zeros_like(image.getArray())
743  convCntrl = afwMath.ConvolutionControl(False, True, 1)
744  fixedKernel = afwMath.FixedKernel(kernelImage)
745 
746  # Define boundary by convolution region. The region that the correction will be
747  # calculated for is one fewer in each dimension because of the second derivative terms.
748  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
749  startX = kLx//2
750  endX = -kLx//2
751  startY = kLy//2
752  endY = -kLy//2
753 
754  for iteration in range(maxIter):
755 
756  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
757  tmpArray = tempImage.getArray()
758  outArray = outImage.getArray()
759 
760  with numpy.errstate(invalid="ignore", over="ignore"):
761  # First derivative term
762  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
763  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
764  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
765 
766  # Second derivative term
767  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
768  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
769  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
770 
771  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
772 
773  tmpArray[:, :] = image.getArray()[:, :]
774  tmpArray[nanIndex] = 0.
775  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
776 
777  if iteration > 0:
778  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
779 
780  if diff < threshold:
781  break
782  prev_image[:, :] = tmpArray[:, :]
783 
784  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
785  corr[startY + 1:endY - 1, startX + 1:endX - 1]
786 
787  return diff, iteration
788 
789 
790 @contextmanager
791 def gainContext(exp, image, apply, gains=None):
792  """Context manager that applies and removes gain.
793 
794  Parameters
795  ----------
796  exp : `lsst.afw.image.Exposure`
797  Exposure to apply/remove gain.
798  image : `lsst.afw.image.Image`
799  Image to apply/remove gain.
800  apply : `Bool`
801  If True, apply and remove the amplifier gain.
802  gains : `dict` [`str`, `float`]
803  A dictionary, keyed by amplifier name, of the gains to use.
804  If gains is None, the nominal gains in the amplifier object are used.
805 
806  Yields
807  ------
808  exp : `lsst.afw.image.Exposure`
809  Exposure with the gain applied.
810  """
811  # check we have all of them if provided because mixing and matching would
812  # be a real mess
813  if gains and apply is True:
814  ampNames = [amp.getName() for amp in exp.getDetector()]
815  for ampName in ampNames:
816  if ampName not in gains.keys():
817  raise RuntimeError(f"Gains provided to gain context, but no entry found for amp {ampName}")
818 
819  if apply:
820  ccd = exp.getDetector()
821  for amp in ccd:
822  sim = image.Factory(image, amp.getBBox())
823  if gains:
824  gain = gains[amp.getName()]
825  else:
826  gain = amp.getGain()
827  sim *= gain
828 
829  try:
830  yield exp
831  finally:
832  if apply:
833  ccd = exp.getDetector()
834  for amp in ccd:
835  sim = image.Factory(image, amp.getBBox())
836  if gains:
837  gain = gains[amp.getName()]
838  else:
839  gain = amp.getGain()
840  sim /= gain
841 
842 
843 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
844  sensorTransmission=None, atmosphereTransmission=None):
845  """Attach a TransmissionCurve to an Exposure, given separate curves for
846  different components.
847 
848  Parameters
849  ----------
850  exposure : `lsst.afw.image.Exposure`
851  Exposure object to modify by attaching the product of all given
852  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
853  Must have a valid ``Detector`` attached that matches the detector
854  associated with sensorTransmission.
855  opticsTransmission : `lsst.afw.image.TransmissionCurve`
856  A ``TransmissionCurve`` that represents the throughput of the optics,
857  to be evaluated in focal-plane coordinates.
858  filterTransmission : `lsst.afw.image.TransmissionCurve`
859  A ``TransmissionCurve`` that represents the throughput of the filter
860  itself, to be evaluated in focal-plane coordinates.
861  sensorTransmission : `lsst.afw.image.TransmissionCurve`
862  A ``TransmissionCurve`` that represents the throughput of the sensor
863  itself, to be evaluated in post-assembly trimmed detector coordinates.
864  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
865  A ``TransmissionCurve`` that represents the throughput of the
866  atmosphere, assumed to be spatially constant.
867 
868  Returns
869  -------
870  combined : `lsst.afw.image.TransmissionCurve`
871  The TransmissionCurve attached to the exposure.
872 
873  Notes
874  -----
875  All ``TransmissionCurve`` arguments are optional; if none are provided, the
876  attached ``TransmissionCurve`` will have unit transmission everywhere.
877  """
878  combined = afwImage.TransmissionCurve.makeIdentity()
879  if atmosphereTransmission is not None:
880  combined *= atmosphereTransmission
881  if opticsTransmission is not None:
882  combined *= opticsTransmission
883  if filterTransmission is not None:
884  combined *= filterTransmission
885  detector = exposure.getDetector()
886  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
887  toSys=camGeom.PIXELS)
888  combined = combined.transformedBy(fpToPix)
889  if sensorTransmission is not None:
890  combined *= sensorTransmission
891  exposure.getInfo().setTransmissionCurve(combined)
892  return combined
893 
894 
895 @deprecated(reason="Camera geometry-based SkyWcs are now set when reading raws. To be removed after v19.",
896  category=FutureWarning)
897 def addDistortionModel(exposure, camera):
898  """!Update the WCS in exposure with a distortion model based on camera
899  geometry.
900 
901  Parameters
902  ----------
903  exposure : `lsst.afw.image.Exposure`
904  Exposure to process. Must contain a Detector and WCS. The
905  exposure is modified.
906  camera : `lsst.afw.cameraGeom.Camera`
907  Camera geometry.
908 
909  Raises
910  ------
911  RuntimeError
912  Raised if ``exposure`` is lacking a Detector or WCS, or if
913  ``camera`` is None.
914  Notes
915  -----
916  Add a model for optical distortion based on geometry found in ``camera``
917  and the ``exposure``'s detector. The raw input exposure is assumed
918  have a TAN WCS that has no compensation for optical distortion.
919  Two other possibilities are:
920  - The raw input exposure already has a model for optical distortion,
921  as is the case for raw DECam data.
922  In that case you should set config.doAddDistortionModel False.
923  - The raw input exposure has a model for distortion, but it has known
924  deficiencies severe enough to be worth fixing (e.g. because they
925  cause problems for fitting a better WCS). In that case you should
926  override this method with a version suitable for your raw data.
927 
928  """
929  wcs = exposure.getWcs()
930  if wcs is None:
931  raise RuntimeError("exposure has no WCS")
932  if camera is None:
933  raise RuntimeError("camera is None")
934  detector = exposure.getDetector()
935  if detector is None:
936  raise RuntimeError("exposure has no Detector")
937  pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE)
938  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE,
939  camGeom.FIELD_ANGLE)
940  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
941  exposure.setWcs(distortedWcs)
942 
943 
944 def applyGains(exposure, normalizeGains=False):
945  """Scale an exposure by the amplifier gains.
946 
947  Parameters
948  ----------
949  exposure : `lsst.afw.image.Exposure`
950  Exposure to process. The image is modified.
951  normalizeGains : `Bool`, optional
952  If True, then amplifiers are scaled to force the median of
953  each amplifier to equal the median of those medians.
954  """
955  ccd = exposure.getDetector()
956  ccdImage = exposure.getMaskedImage()
957 
958  medians = []
959  for amp in ccd:
960  sim = ccdImage.Factory(ccdImage, amp.getBBox())
961  sim *= amp.getGain()
962 
963  if normalizeGains:
964  medians.append(numpy.median(sim.getImage().getArray()))
965 
966  if normalizeGains:
967  median = numpy.median(numpy.array(medians))
968  for index, amp in enumerate(ccd):
969  sim = ccdImage.Factory(ccdImage, amp.getBBox())
970  if medians[index] != 0.0:
971  sim *= median/medians[index]
972 
973 
975  """Grow the saturation trails by an amount dependent on the width of the trail.
976 
977  Parameters
978  ----------
979  mask : `lsst.afw.image.Mask`
980  Mask which will have the saturated areas grown.
981  """
982 
983  extraGrowDict = {}
984  for i in range(1, 6):
985  extraGrowDict[i] = 0
986  for i in range(6, 8):
987  extraGrowDict[i] = 1
988  for i in range(8, 10):
989  extraGrowDict[i] = 3
990  extraGrowMax = 4
991 
992  if extraGrowMax <= 0:
993  return
994 
995  saturatedBit = mask.getPlaneBitMask("SAT")
996 
997  xmin, ymin = mask.getBBox().getMin()
998  width = mask.getWidth()
999 
1000  thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK)
1001  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
1002 
1003  for fp in fpList:
1004  for s in fp.getSpans():
1005  x0, x1 = s.getX0(), s.getX1()
1006 
1007  extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
1008  if extraGrow > 0:
1009  y = s.getY() - ymin
1010  x0 -= xmin + extraGrow
1011  x1 -= xmin - extraGrow
1012 
1013  if x0 < 0:
1014  x0 = 0
1015  if x1 >= width - 1:
1016  x1 = width - 1
1017 
1018  mask.array[y, x0:x1+1] |= saturatedBit
1019 
1020 
1021 def setBadRegions(exposure, badStatistic="MEDIAN"):
1022  """Set all BAD areas of the chip to the average of the rest of the exposure
1023 
1024  Parameters
1025  ----------
1026  exposure : `lsst.afw.image.Exposure`
1027  Exposure to mask. The exposure mask is modified.
1028  badStatistic : `str`, optional
1029  Statistic to use to generate the replacement value from the
1030  image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
1031 
1032  Returns
1033  -------
1034  badPixelCount : scalar
1035  Number of bad pixels masked.
1036  badPixelValue : scalar
1037  Value substituted for bad pixels.
1038 
1039  Raises
1040  ------
1041  RuntimeError
1042  Raised if `badStatistic` is not an allowed value.
1043  """
1044  if badStatistic == "MEDIAN":
1045  statistic = afwMath.MEDIAN
1046  elif badStatistic == "MEANCLIP":
1047  statistic = afwMath.MEANCLIP
1048  else:
1049  raise RuntimeError("Impossible method %s of bad region correction" % badStatistic)
1050 
1051  mi = exposure.getMaskedImage()
1052  mask = mi.getMask()
1053  BAD = mask.getPlaneBitMask("BAD")
1054  INTRP = mask.getPlaneBitMask("INTRP")
1055 
1056  sctrl = afwMath.StatisticsControl()
1057  sctrl.setAndMask(BAD)
1058  value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
1059 
1060  maskArray = mask.getArray()
1061  imageArray = mi.getImage().getArray()
1062  badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0)
1063  imageArray[:] = numpy.where(badPixels, value, imageArray)
1064 
1065  return badPixels.sum(), value
def addDistortionModel(exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def setBadRegions(exposure, badStatistic="MEDIAN")
def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1, maskNameList=['SAT'], fallbackValue=None)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:78
def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain, gains=None)
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:58
def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def growMasks(mask, radius=0, maskNameList=['BAD'], maskValue="BAD")
def applyGains(exposure, normalizeGains=False)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False)
def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False)
def gainContext(exp, image, apply, gains=None)
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale, trimToFit=True)
def updateVariance(maskedImage, gain, readNoise)
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None, overscanIsInt=True)