lsst.ip.isr  7.9-hsc
isrFunctions.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008, 2009, 2010 LSST Corporation.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 import math
23 import numpy
24 
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.detection as afwDetection
28 import lsst.afw.math as afwMath
29 import lsst.meas.algorithms as measAlg
30 import lsst.pex.exceptions as pexExcept
31 import lsst.afw.cameraGeom as camGeom
32 
33 from lsst.afw.geom.wcsUtils import makeDistortedTanWcs
34 from lsst.meas.algorithms.detection import SourceDetectionTask
35 from lsst.pipe.base import Struct
36 
37 from contextlib import contextmanager
38 
39 
40 def createPsf(fwhm):
41  """Make a double Gaussian PSF.
42 
43  Parameters
44  ----------
45  fwhm : scalar
46  FWHM of double Gaussian smoothing kernel.
47 
48  Returns
49  -------
50  psf : `lsst.meas.algorithms.DoubleGaussianPsf`
51  The created smoothing kernel.
52  """
53  ksize = 4*int(fwhm) + 1
54  return measAlg.DoubleGaussianPsf(ksize, ksize, fwhm/(2*math.sqrt(2*math.log(2))))
55 
56 
57 def transposeMaskedImage(maskedImage):
58  """Make a transposed copy of a masked image.
59 
60  Parameters
61  ----------
62  maskedImage : `lsst.afw.image.MaskedImage`
63  Image to process.
64 
65  Returns
66  -------
67  transposed : `lsst.afw.image.MaskedImage`
68  The transposed copy of the input image.
69  """
70  transposed = maskedImage.Factory(afwGeom.Extent2I(maskedImage.getHeight(), maskedImage.getWidth()))
71  transposed.getImage().getArray()[:] = maskedImage.getImage().getArray().T
72  transposed.getMask().getArray()[:] = maskedImage.getMask().getArray().T
73  transposed.getVariance().getArray()[:] = maskedImage.getVariance().getArray().T
74  return transposed
75 
76 
77 def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None):
78  """Interpolate over defects specified in a defect list.
79 
80  Parameters
81  ----------
82  maskedImage : `lsst.afw.image.MaskedImage`
83  Image to process.
84  defectList : `list`
85  List of defects to interpolate over.
86  fwhm : scalar
87  FWHM of double Gaussian smoothing kernel.
88  fallbackValue : scalar, optional
89  Fallback value if an interpolated value cannot be determined.
90  If None, then the clipped mean of the image is used.
91  """
92  psf = createPsf(fwhm)
93  if fallbackValue is None:
94  fallbackValue = afwMath.makeStatistics(maskedImage.getImage(), afwMath.MEANCLIP).getValue()
95  if 'INTRP' not in maskedImage.getMask().getMaskPlaneDict():
96  maskedImage.getMask.addMaskPlane('INTRP')
97  measAlg.interpolateOverDefects(maskedImage, psf, defectList, fallbackValue, True)
98 
99 
101  """Compute a defect list from a footprint list, optionally growing the footprints.
102 
103  Parameters
104  ----------
105  fpList : `list` of `lsst.afw.detection.Footprint`
106  Footprint list to process.
107 
108  Returns
109  -------
110  defectList : `list` of `lsst.afw.meas.algorithms.Defect`
111  List of defects.
112  """
113  defectList = []
114  for fp in fpList:
115  for bbox in afwDetection.footprintToBBoxList(fp):
116  defect = measAlg.Defect(bbox)
117  defectList.append(defect)
118  return defectList
119 
120 
121 def transposeDefectList(defectList):
122  """Make a transposed copy of a defect list.
123 
124  Parameters
125  ----------
126  defectList : `list` of `lsst.afw.meas.algorithms.Defect`
127  Input list of defects.
128 
129  Returns
130  -------
131  retDefectList : `list` of `lsst.afw.meas.algorithms.Defect`
132  Transposed list of defects.
133  """
134  retDefectList = []
135  for defect in defectList:
136  bbox = defect.getBBox()
137  nbbox = afwGeom.Box2I(afwGeom.Point2I(bbox.getMinY(), bbox.getMinX()),
138  afwGeom.Extent2I(bbox.getDimensions()[1], bbox.getDimensions()[0]))
139  retDefectList.append(measAlg.Defect(nbbox))
140  return retDefectList
141 
142 
143 def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD'):
144  """Set mask plane based on a defect list.
145 
146  Parameters
147  ----------
148  maskedImage : `lsst.afw.image.MaskedImage`
149  Image to process. Only the mask plane is updated.
150  defectList : `list` of `lsst.afw.meas.algorithms.Defect`
151  Defect list to mask.
152  maskName : str, optional
153  Mask plane name to use.
154  """
155  # mask bad pixels
156  mask = maskedImage.getMask()
157  bitmask = mask.getPlaneBitMask(maskName)
158  for defect in defectList:
159  bbox = defect.getBBox()
160  afwGeom.SpanSet(bbox).clippedTo(mask.getBBox()).setMask(mask, bitmask)
161 
162 
163 def getDefectListFromMask(maskedImage, maskName):
164  """Compute a defect list from a specified mask plane.
165 
166  Parameters
167  ----------
168  maskedImage : `lsst.afw.image.MaskedImage`
169  Image to process.
170  maskName : str or `list`
171  Mask plane name, or list of names to convert.
172 
173  Returns
174  -------
175  defectList : `list` of `lsst.afw.meas.algorithms.Defect`
176  Defect list constructed from masked pixels.
177  """
178  mask = maskedImage.getMask()
179  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskName), afwDetection.Threshold.BITMASK)
180  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
181  return defectListFromFootprintList(fpList)
182 
183 
184 def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT'):
185  """Mask pixels based on threshold detection.
186 
187  Parameters
188  ----------
189  maskedImage : `lsst.afw.image.MaskedImage`
190  Image to process. Only the mask plane is updated.
191  threshold : scalar
192  Detection threshold.
193  growFootprints : scalar, optional
194  Number of pixels to grow footprints of detected regions.
195  maskName : str, optional
196  Mask plane name, or list of names to convert
197 
198  Returns
199  -------
200  defectList : `list` of `lsst.afw.meas.algorithms.Defect`
201  Defect list constructed from pixels above the threshold.
202  """
203  # find saturated regions
204  thresh = afwDetection.Threshold(threshold)
205  fs = afwDetection.FootprintSet(maskedImage, thresh)
206 
207  if growFootprints > 0:
208  fs = afwDetection.FootprintSet(fs, growFootprints)
209  fpList = fs.getFootprints()
210 
211  # set mask
212  mask = maskedImage.getMask()
213  bitmask = mask.getPlaneBitMask(maskName)
214  afwDetection.setMaskFromFootprintList(mask, fpList, bitmask)
215 
216  return defectListFromFootprintList(fpList)
217 
218 
219 def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1,
220  maskNameList=['SAT'], fallbackValue=None):
221  """Interpolate over defects identified by a particular set of mask planes.
222 
223  Parameters
224  ----------
225  maskedImage : `lsst.afw.image.MaskedImage`
226  Image to process.
227  fwhm : scalar
228  FWHM of double Gaussian smoothing kernel.
229  growSaturatedFootprints : scalar, optional
230  Number of pixels to grow footprints for saturated pixels.
231  maskNameList : `List` of `str`, optional
232  Mask plane name.
233  fallbackValue : scalar, optional
234  Value of last resort for interpolation.
235  """
236  mask = maskedImage.getMask()
237 
238  if growSaturatedFootprints > 0 and "SAT" in maskNameList:
239  thresh = afwDetection.Threshold(mask.getPlaneBitMask("SAT"), afwDetection.Threshold.BITMASK)
240  fpSet = afwDetection.FootprintSet(mask, thresh)
241  # If we are interpolating over an area larger than the original masked region, we need
242  # to expand the original mask bit to the full area to explain why we interpolated there.
243  fpSet = afwDetection.FootprintSet(fpSet, rGrow=growSaturatedFootprints, isotropic=False)
244  fpSet.setMask(mask, "SAT")
245 
246  thresh = afwDetection.Threshold(mask.getPlaneBitMask(maskNameList), afwDetection.Threshold.BITMASK)
247  fpSet = afwDetection.FootprintSet(mask, thresh)
248 
249  defectList = defectListFromFootprintList(fpSet.getFootprints())
250 
251  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
252 
253 
254 def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT',
255  fallbackValue=None):
256  """Mark saturated pixels and optionally interpolate over them
257 
258  Parameters
259  ----------
260  maskedImage : `lsst.afw.image.MaskedImage`
261  Image to process.
262  saturation : scalar
263  Saturation level used as the detection threshold.
264  fwhm : scalar
265  FWHM of double Gaussian smoothing kernel.
266  growFootprints : scalar, optional
267  Number of pixels to grow footprints of detected regions.
268  interpolate : Bool, optional
269  If True, saturated pixels are interpolated over.
270  maskName : str, optional
271  Mask plane name.
272  fallbackValue : scalar, optional
273  Value of last resort for interpolation.
274  """
275  defectList = makeThresholdMask(
276  maskedImage=maskedImage,
277  threshold=saturation,
278  growFootprints=growFootprints,
279  maskName=maskName,
280  )
281  if interpolate:
282  interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=fallbackValue)
283 
284 
285 def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage):
286  """Compute number of edge trim pixels to match the calibration data.
287 
288  Use the dimension difference between the raw exposure and the
289  calibration exposure to compute the edge trim pixels. This trim
290  is applied symmetrically, with the same number of pixels masked on
291  each side.
292 
293  Parameters
294  ----------
295  rawMaskedImage : `lsst.afw.image.MaskedImage`
296  Image to trim.
297  calibMaskedImage : `lsst.afw.image.MaskedImage`
298  Calibration image to draw new bounding box from.
299 
300  Returns
301  -------
302  replacementMaskedImage : `lsst.afw.image.MaskedImage`
303  ``rawMaskedImage`` trimmed to the appropriate size
304  Raises
305  ------
306  RuntimeError
307  Rasied if ``rawMaskedImage`` cannot be symmetrically trimmed to
308  match ``calibMaskedImage``.
309  """
310  nx, ny = rawMaskedImage.getBBox().getDimensions() - calibMaskedImage.getBBox().getDimensions()
311  if nx != ny:
312  raise RuntimeError("Raw and calib maskedImages are trimmed differently in X and Y.")
313  if nx % 2 != 0:
314  raise RuntimeError("Calibration maskedImage is trimmed unevenly in X.")
315  if nx <= 0:
316  raise RuntimeError("Calibration maskedImage is larger than raw data.")
317 
318  nEdge = nx//2
319  if nEdge > 0:
320  replacementMaskedImage = rawMaskedImage[nEdge:-nEdge, nEdge:-nEdge, afwImage.LOCAL]
321  SourceDetectionTask.setEdgeBits(
322  rawMaskedImage,
323  replacementMaskedImage.getBBox(),
324  rawMaskedImage.getMask().getPlaneBitMask("EDGE")
325  )
326  else:
327  replacementMaskedImage = rawMaskedImage
328 
329  return replacementMaskedImage
330 
331 
332 def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False):
333  """Apply bias correction in place.
334 
335  Parameters
336  ----------
337  maskedImage : `lsst.afw.image.MaskedImage`
338  Image to process. The image is modified by this method.
339  biasMaskedImage : `lsst.afw.image.MaskedImage`
340  Bias image of the same size as ``maskedImage``
341  trimToFit : `Bool`, optional
342  If True, raw data is symmetrically trimmed to match
343  calibration size.
344 
345  Raises
346  ------
347  RuntimeError
348  Raised if ``maskedImage`` and ``biasMaskedImage`` do not have
349  the same size.
350 
351  """
352  if trimToFit:
353  maskedImage = trimToMatchCalibBBox(maskedImage, biasMaskedImage)
354 
355  if maskedImage.getBBox(afwImage.LOCAL) != biasMaskedImage.getBBox(afwImage.LOCAL):
356  raise RuntimeError("maskedImage bbox %s != biasMaskedImage bbox %s" %
357  (maskedImage.getBBox(afwImage.LOCAL), biasMaskedImage.getBBox(afwImage.LOCAL)))
358  maskedImage -= biasMaskedImage
359 
360 
361 def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False):
362  """Apply dark correction in place.
363 
364  Parameters
365  ----------
366  maskedImage : `lsst.afw.image.MaskedImage`
367  Image to process. The image is modified by this method.
368  darkMaskedImage : `lsst.afw.image.MaskedImage`
369  Dark image of the same size as ``maskedImage``.
370  expScale : scalar
371  Dark exposure time for ``maskedImage``.
372  darkScale : scalar
373  Dark exposure time for ``darkMaskedImage``.
374  invert : `Bool`, optional
375  If True, re-add the dark to an already corrected image.
376  trimToFit : `Bool`, optional
377  If True, raw data is symmetrically trimmed to match
378  calibration size.
379 
380  Raises
381  ------
382  RuntimeError
383  Raised if ``maskedImage`` and ``darkMaskedImage`` do not have
384  the same size.
385 
386  Notes
387  -----
388  The dark correction is applied by calculating:
389  maskedImage -= dark * expScaling / darkScaling
390  """
391  if trimToFit:
392  maskedImage = trimToMatchCalibBBox(maskedImage, darkMaskedImage)
393 
394  if maskedImage.getBBox(afwImage.LOCAL) != darkMaskedImage.getBBox(afwImage.LOCAL):
395  raise RuntimeError("maskedImage bbox %s != darkMaskedImage bbox %s" %
396  (maskedImage.getBBox(afwImage.LOCAL), darkMaskedImage.getBBox(afwImage.LOCAL)))
397 
398  scale = expScale / darkScale
399  if not invert:
400  maskedImage.scaledMinus(scale, darkMaskedImage)
401  else:
402  maskedImage.scaledPlus(scale, darkMaskedImage)
403 
404 
405 def updateVariance(maskedImage, gain, readNoise):
406  """Set the variance plane based on the image plane.
407 
408  Parameters
409  ----------
410  maskedImage : `lsst.afw.image.MaskedImage`
411  Image to process. The variance plane is modified.
412  gain : scalar
413  The amplifier gain in electrons/ADU.
414  readNoise : scalar
415  The amplifier read nmoise in ADU/pixel.
416  """
417  var = maskedImage.getVariance()
418  var[:] = maskedImage.getImage()
419  var /= gain
420  var += readNoise**2
421 
422 
423 def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False):
424  """Apply flat correction in place.
425 
426  Parameters
427  ----------
428  maskedImage : `lsst.afw.image.MaskedImage`
429  Image to process. The image is modified.
430  flatMaskedImage : `lsst.afw.image.MaskedImage`
431  Flat image of the same size as ``maskedImage``
432  scalingType : str
433  Flat scale computation method. Allowed values are 'MEAN',
434  'MEDIAN', or 'USER'.
435  userScale : scalar, optional
436  Scale to use if ``scalingType``='USER'.
437  invert : `Bool`, optional
438  If True, unflatten an already flattened image.
439  trimToFit : `Bool`, optional
440  If True, raw data is symmetrically trimmed to match
441  calibration size.
442 
443  Raises
444  ------
445  RuntimeError
446  Raised if ``maskedImage`` and ``flatMaskedImage`` do not have
447  the same size.
448  pexExcept.Exception
449  Raised if ``scalingType`` is not an allowed value.
450  """
451  if trimToFit:
452  maskedImage = trimToMatchCalibBBox(maskedImage, flatMaskedImage)
453 
454  if maskedImage.getBBox(afwImage.LOCAL) != flatMaskedImage.getBBox(afwImage.LOCAL):
455  raise RuntimeError("maskedImage bbox %s != flatMaskedImage bbox %s" %
456  (maskedImage.getBBox(afwImage.LOCAL), flatMaskedImage.getBBox(afwImage.LOCAL)))
457 
458  # Figure out scale from the data
459  # Ideally the flats are normalized by the calibration product pipeline, but this allows some flexibility
460  # in the case that the flat is created by some other mechanism.
461  if scalingType in ('MEAN', 'MEDIAN'):
462  scalingType = afwMath.stringToStatisticsProperty(scalingType)
463  flatScale = afwMath.makeStatistics(flatMaskedImage.image, scalingType).getValue()
464  elif scalingType == 'USER':
465  flatScale = userScale
466  else:
467  raise pexExcept.Exception('%s : %s not implemented' % ("flatCorrection", scalingType))
468 
469  if not invert:
470  maskedImage.scaledDivides(1.0/flatScale, flatMaskedImage)
471  else:
472  maskedImage.scaledMultiplies(1.0/flatScale, flatMaskedImage)
473 
474 
475 def illuminationCorrection(maskedImage, illumMaskedImage, illumScale):
476  """Apply illumination correction in place.
477 
478  Parameters
479  ----------
480  maskedImage : `lsst.afw.image.MaskedImage`
481  Image to process. The image is modified.
482  illumMaskedImage : `lsst.afw.image.MaskedImage`
483  Illumination correction image of the same size as ``maskedImage``.
484  illumScale : scalar
485  Scale factor for the illumination correction.
486 
487  Raises
488  ------
489  RuntimeError
490  Raised if ``maskedImage`` and ``illumMaskedImage`` do not have
491  the same size.
492  """
493  if maskedImage.getBBox(afwImage.LOCAL) != illumMaskedImage.getBBox(afwImage.LOCAL):
494  raise RuntimeError("maskedImage bbox %s != illumMaskedImage bbox %s" %
495  (maskedImage.getBBox(afwImage.LOCAL), illumMaskedImage.getBBox(afwImage.LOCAL)))
496 
497  maskedImage.scaledDivides(1./illumScale, illumMaskedImage)
498 
499 
500 def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0,
501  statControl=None, overscanIsInt=True):
502  """Apply overscan correction in place.
503 
504  Parameters
505  ----------
506  ampMaskedImage : `lsst.afw.image.MaskedImage`
507  Image of amplifier to correct; modified.
508  overscanImage : `lsst.afw.image.Image` or `lsst.afw.image.MaskedImage`
509  Image of overscan; modified.
510  fitType : `str`
511  Type of fit for overscan correction. May be one of:
512 
513  - ``MEAN``: use mean of overscan.
514  - ``MEANCLIP``: use clipped mean of overscan.
515  - ``MEDIAN``: use median of overscan.
516  - ``POLY``: fit with ordinary polynomial.
517  - ``CHEB``: fit with Chebyshev polynomial.
518  - ``LEG``: fit with Legendre polynomial.
519  - ``NATURAL_SPLINE``: fit with natural spline.
520  - ``CUBIC_SPLINE``: fit with cubic spline.
521  - ``AKIMA_SPLINE``: fit with Akima spline.
522 
523  order : `int`
524  Polynomial order or number of spline knots; ignored unless
525  ``fitType`` indicates a polynomial or spline.
526  statControl : `lsst.afw.math.StatisticsControl`
527  Statistics control object. In particular, we pay attention to numSigmaClip
528  overscanIsInt : `bool`
529  Treat the overscan region as consisting of integers, even if it's been
530  converted to float. E.g. handle ties properly.
531 
532  Returns
533  -------
534  result : `lsst.pipe.base.Struct`
535  Result struct with components:
536 
537  - ``imageFit``: Value(s) removed from image (scalar or
538  `lsst.afw.image.Image`)
539  - ``overscanFit``: Value(s) removed from overscan (scalar or
540  `lsst.afw.image.Image`)
541  - ``overscanImage``: Overscan corrected overscan region
542  (`lsst.afw.image.Image`)
543  Raises
544  ------
545  pexExcept.Exception
546  Raised if ``fitType`` is not an allowed value.
547 
548  Notes
549  -----
550  The ``ampMaskedImage`` and ``overscanImage`` are modified, with the fit
551  subtracted. Note that the ``overscanImage`` should not be a subimage of
552  the ``ampMaskedImage``, to avoid being subtracted twice.
553 
554  Debug plots are available for the SPLINE fitTypes by setting the
555  `debug.display` for `name` == "lsst.ip.isr.isrFunctions". These
556  plots show the scatter plot of the overscan data (collapsed along
557  the perpendicular dimension) as a function of position on the CCD
558  (normalized between +/-1).
559  """
560  ampImage = ampMaskedImage.getImage()
561  if statControl is None:
562  statControl = afwMath.StatisticsControl()
563 
564  numSigmaClip = statControl.getNumSigmaClip()
565 
566  if fitType in ('MEAN', 'MEANCLIP'):
567  fitType = afwMath.stringToStatisticsProperty(fitType)
568  offImage = afwMath.makeStatistics(overscanImage, fitType, statControl).getValue()
569  overscanFit = offImage
570  elif fitType in ('MEDIAN',):
571  if overscanIsInt:
572  # we need an image with integer pixels to handle ties properly
573  if hasattr(overscanImage, "image"):
574  imageI = overscanImage.image.convertI()
575  overscanImageI = afwImage.MaskedImageI(imageI, overscanImage.mask, overscanImage.variance)
576  else:
577  overscanImageI = overscanImage.convertI()
578  else:
579  overscanImageI = overscanImage
580 
581  fitType = afwMath.stringToStatisticsProperty(fitType)
582  offImage = afwMath.makeStatistics(overscanImageI, fitType, statControl).getValue()
583  overscanFit = offImage
584 
585  if overscanIsInt:
586  del overscanImageI
587  elif fitType in ('POLY', 'CHEB', 'LEG', 'NATURAL_SPLINE', 'CUBIC_SPLINE', 'AKIMA_SPLINE'):
588  if hasattr(overscanImage, "getImage"):
589  biasArray = overscanImage.getImage().getArray()
590  biasArray = numpy.ma.masked_where(overscanImage.getMask().getArray() & statControl.getAndMask(),
591  biasArray)
592  else:
593  biasArray = overscanImage.getArray()
594  # Fit along the long axis, so collapse along each short row and fit the resulting array
595  shortInd = numpy.argmin(biasArray.shape)
596  if shortInd == 0:
597  # Convert to some 'standard' representation to make things easier
598  biasArray = numpy.transpose(biasArray)
599 
600  # Do a single round of clipping to weed out CR hits and signal leaking into the overscan
601  percentiles = numpy.percentile(biasArray, [25.0, 50.0, 75.0], axis=1)
602  medianBiasArr = percentiles[1]
603  stdevBiasArr = 0.74*(percentiles[2] - percentiles[0]) # robust stdev
604  diff = numpy.abs(biasArray - medianBiasArr[:, numpy.newaxis])
605  biasMaskedArr = numpy.ma.masked_where(diff > numSigmaClip*stdevBiasArr[:, numpy.newaxis], biasArray)
606  collapsed = numpy.mean(biasMaskedArr, axis=1)
607  if collapsed.mask.sum() > 0:
608  collapsed.data[collapsed.mask] = numpy.mean(biasArray.data[collapsed.mask], axis=1)
609  del biasArray, percentiles, stdevBiasArr, diff, biasMaskedArr
610 
611  if shortInd == 0:
612  collapsed = numpy.transpose(collapsed)
613 
614  num = len(collapsed)
615  indices = 2.0*numpy.arange(num)/float(num) - 1.0
616 
617  if fitType in ('POLY', 'CHEB', 'LEG'):
618  # A numpy polynomial
619  poly = numpy.polynomial
620  fitter, evaler = {"POLY": (poly.polynomial.polyfit, poly.polynomial.polyval),
621  "CHEB": (poly.chebyshev.chebfit, poly.chebyshev.chebval),
622  "LEG": (poly.legendre.legfit, poly.legendre.legval),
623  }[fitType]
624 
625  coeffs = fitter(indices, collapsed, order)
626  fitBiasArr = evaler(indices, coeffs)
627  elif 'SPLINE' in fitType:
628  # An afw interpolation
629  numBins = order
630  #
631  # numpy.histogram needs a real array for the mask, but numpy.ma "optimises" the case
632  # no-values-are-masked by replacing the mask array by a scalar, numpy.ma.nomask
633  #
634  # Issue DM-415
635  #
636  collapsedMask = collapsed.mask
637  try:
638  if collapsedMask == numpy.ma.nomask:
639  collapsedMask = numpy.array(len(collapsed)*[numpy.ma.nomask])
640  except ValueError: # If collapsedMask is an array the test fails [needs .all()]
641  pass
642 
643  numPerBin, binEdges = numpy.histogram(indices, bins=numBins,
644  weights=1-collapsedMask.astype(int))
645  # Binning is just a histogram, with weights equal to the values.
646  # Use a similar trick to get the bin centers (this deals with different numbers per bin).
647  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
648  values = numpy.histogram(indices, bins=numBins,
649  weights=collapsed.data*~collapsedMask)[0]/numPerBin
650  binCenters = numpy.histogram(indices, bins=numBins,
651  weights=indices*~collapsedMask)[0]/numPerBin
652  interp = afwMath.makeInterpolate(binCenters.astype(float)[numPerBin > 0],
653  values.astype(float)[numPerBin > 0],
654  afwMath.stringToInterpStyle(fitType))
655  fitBiasArr = numpy.array([interp.interpolate(i) for i in indices])
656 
657  import lsstDebug
658  if lsstDebug.Info(__name__).display:
659  import matplotlib.pyplot as plot
660  figure = plot.figure(1)
661  figure.clear()
662  axes = figure.add_axes((0.1, 0.1, 0.8, 0.8))
663  axes.plot(indices[~collapsedMask], collapsed[~collapsedMask], 'k+')
664  if collapsedMask.sum() > 0:
665  axes.plot(indices[collapsedMask], collapsed.data[collapsedMask], 'b+')
666  axes.plot(indices, fitBiasArr, 'r-')
667  plot.xlabel("centered/scaled position along overscan region")
668  plot.ylabel("pixel value/fit value")
669  figure.show()
670  prompt = "Press Enter or c to continue [chp]... "
671  while True:
672  ans = input(prompt).lower()
673  if ans in ("", "c",):
674  break
675  if ans in ("p",):
676  import pdb
677  pdb.set_trace()
678  elif ans in ("h", ):
679  print("h[elp] c[ontinue] p[db]")
680  plot.close()
681 
682  offImage = ampImage.Factory(ampImage.getDimensions())
683  offArray = offImage.getArray()
684  overscanFit = afwImage.ImageF(overscanImage.getDimensions())
685  overscanArray = overscanFit.getArray()
686  if shortInd == 1:
687  offArray[:, :] = fitBiasArr[:, numpy.newaxis]
688  overscanArray[:, :] = fitBiasArr[:, numpy.newaxis]
689  else:
690  offArray[:, :] = fitBiasArr[numpy.newaxis, :]
691  overscanArray[:, :] = fitBiasArr[numpy.newaxis, :]
692 
693  # We don't trust any extrapolation: mask those pixels as SUSPECT
694  # This will occur when the top and or bottom edges of the overscan
695  # contain saturated values. The values will be extrapolated from
696  # the surrounding pixels, but we cannot entirely trust the value of
697  # the extrapolation, and will mark the image mask plane to flag the
698  # image as such.
699  mask = ampMaskedImage.getMask()
700  maskArray = mask.getArray() if shortInd == 1 else mask.getArray().transpose()
701  suspect = mask.getPlaneBitMask("SUSPECT")
702  try:
703  if collapsed.mask == numpy.ma.nomask:
704  # There is no mask, so the whole array is fine
705  pass
706  except ValueError: # If collapsed.mask is an array the test fails [needs .all()]
707  for low in range(num):
708  if not collapsed.mask[low]:
709  break
710  if low > 0:
711  maskArray[:low, :] |= suspect
712  for high in range(1, num):
713  if not collapsed.mask[-high]:
714  break
715  if high > 1:
716  maskArray[-high:, :] |= suspect
717 
718  else:
719  raise pexExcept.Exception('%s : %s an invalid overscan type' % ("overscanCorrection", fitType))
720  ampImage -= offImage
721  overscanImage -= overscanFit
722  return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
723 
724 
725 def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
726  """Apply brighter fatter correction in place for the image.
727 
728  Parameters
729  ----------
730  exposure : `lsst.afw.image.Exposure`
731  Exposure to have brighter-fatter correction applied. Modified
732  by this method.
733  kernel : `numpy.ndarray`
734  Brighter-fatter kernel to apply.
735  maxIter : scalar
736  Number of correction iterations to run.
737  threshold : scalar
738  Convergence threshold in terms of the sum of absolute
739  deviations between an iteration and the previous one.
740  applyGain : `Bool`
741  If True, then the exposure values are scaled by the gain prior
742  to correction.
743 
744  Notes
745  -----
746  This correction takes a kernel that has been derived from flat
747  field images to redistribute the charge. The gradient of the
748  kernel is the deflection field due to the accumulated charge.
749 
750  Given the original image I(x) and the kernel K(x) we can compute
751  the corrected image Ic(x) using the following equation:
752 
753  Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y))))
754 
755  To evaluate the derivative term we expand it as follows:
756 
757  0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) )
758 
759  Because we use the measured counts instead of the incident counts
760  we apply the correction iteratively to reconstruct the original
761  counts and the correction. We stop iterating when the summed
762  difference between the current corrected image and the one from
763  the previous iteration is below the threshold. We do not require
764  convergence because the number of iterations is too large a
765  computational cost. How we define the threshold still needs to be
766  evaluated, the current default was shown to work reasonably well
767  on a small set of images. For more information on the method see
768  DocuShare Document-19407.
769 
770  The edges as defined by the kernel are not corrected because they
771  have spurious values due to the convolution.
772  """
773  image = exposure.getMaskedImage().getImage()
774 
775  # The image needs to be units of electrons/holes
776  with gainContext(exposure, image, applyGain):
777 
778  kLx = numpy.shape(kernel)[0]
779  kLy = numpy.shape(kernel)[1]
780  kernelImage = afwImage.ImageD(kLx, kLy)
781  kernelImage.getArray()[:, :] = kernel
782  tempImage = image.clone()
783 
784  nanIndex = numpy.isnan(tempImage.getArray())
785  tempImage.getArray()[nanIndex] = 0.
786 
787  outImage = afwImage.ImageF(image.getDimensions())
788  corr = numpy.zeros_like(image.getArray())
789  prev_image = numpy.zeros_like(image.getArray())
790  convCntrl = afwMath.ConvolutionControl(False, True, 1)
791  fixedKernel = afwMath.FixedKernel(kernelImage)
792 
793  # Define boundary by convolution region. The region that the correction will be
794  # calculated for is one fewer in each dimension because of the second derivative terms.
795  # NOTE: these need to use integer math, as we're using start:end as numpy index ranges.
796  startX = kLx//2
797  endX = -kLx//2
798  startY = kLy//2
799  endY = -kLy//2
800 
801  for iteration in range(maxIter):
802 
803  afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl)
804  tmpArray = tempImage.getArray()
805  outArray = outImage.getArray()
806 
807  with numpy.errstate(invalid="ignore", over="ignore"):
808  # First derivative term
809  gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX])
810  gradOut = numpy.gradient(outArray[startY:endY, startX:endX])
811  first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1]
812 
813  # Second derivative term
814  diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1]
815  diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX]
816  second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21)
817 
818  corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second)
819 
820  tmpArray[:, :] = image.getArray()[:, :]
821  tmpArray[nanIndex] = 0.
822  tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX]
823 
824  if iteration > 0:
825  diff = numpy.sum(numpy.abs(prev_image - tmpArray))
826 
827  if diff < threshold:
828  break
829  prev_image[:, :] = tmpArray[:, :]
830 
831  # if iteration == maxIter - 1:
832  # self.log.warn("Brighter fatter correction did not converge,
833  # final difference %f" % diff)
834 
835  # self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1))
836  image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \
837  corr[startY + 1:endY - 1, startX + 1:endX - 1]
838 
839 
840 @contextmanager
841 def gainContext(exp, image, apply):
842  """Context manager that applies and removes gain.
843 
844  Parameters
845  ----------
846  exp : `lsst.afw.image.Exposure`
847  Exposure to apply/remove gain.
848  image : `lsst.afw.image.Image`
849  Image to apply/remove gain.
850  apply : `Bool`
851  If True, apply and remove the amplifier gain.
852 
853  Yields
854  ------
855  exp : `lsst.afw.image.Exposure`
856  Exposure with the gain applied.
857  """
858  if apply:
859  ccd = exp.getDetector()
860  for amp in ccd:
861  sim = image.Factory(image, amp.getBBox())
862  sim *= amp.getGain()
863 
864  try:
865  yield exp
866  finally:
867  if apply:
868  ccd = exp.getDetector()
869  for amp in ccd:
870  sim = image.Factory(image, amp.getBBox())
871  sim /= amp.getGain()
872 
873 
874 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,
875  sensorTransmission=None, atmosphereTransmission=None):
876  """Attach a TransmissionCurve to an Exposure, given separate curves for
877  different components.
878 
879  Parameters
880  ----------
881  exposure : `lsst.afw.image.Exposure`
882  Exposure object to modify by attaching the product of all given
883  ``TransmissionCurves`` in post-assembly trimmed detector coordinates.
884  Must have a valid ``Detector`` attached that matches the detector
885  associated with sensorTransmission.
886  opticsTransmission : `lsst.afw.image.TransmissionCurve`
887  A ``TransmissionCurve`` that represents the throughput of the optics,
888  to be evaluated in focal-plane coordinates.
889  filterTransmission : `lsst.afw.image.TransmissionCurve`
890  A ``TransmissionCurve`` that represents the throughput of the filter
891  itself, to be evaluated in focal-plane coordinates.
892  sensorTransmission : `lsst.afw.image.TransmissionCurve`
893  A ``TransmissionCurve`` that represents the throughput of the sensor
894  itself, to be evaluated in post-assembly trimmed detector coordinates.
895  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
896  A ``TransmissionCurve`` that represents the throughput of the
897  atmosphere, assumed to be spatially constant.
898 
899  Returns
900  -------
901  combined : `lsst.afw.image.TransmissionCurve`
902  The TransmissionCurve attached to the exposure.
903 
904  Notes
905  -----
906  All ``TransmissionCurve`` arguments are optional; if none are provided, the
907  attached ``TransmissionCurve`` will have unit transmission everywhere.
908  """
909  combined = afwImage.TransmissionCurve.makeIdentity()
910  if atmosphereTransmission is not None:
911  combined *= atmosphereTransmission
912  if opticsTransmission is not None:
913  combined *= opticsTransmission
914  if filterTransmission is not None:
915  combined *= filterTransmission
916  detector = exposure.getDetector()
917  fpToPix = detector.getTransform(fromSys=camGeom.FOCAL_PLANE,
918  toSys=camGeom.PIXELS)
919  combined = combined.transformedBy(fpToPix)
920  if sensorTransmission is not None:
921  combined *= sensorTransmission
922  exposure.getInfo().setTransmissionCurve(combined)
923  return combined
924 
925 
926 def addDistortionModel(exposure, camera):
927  """!Update the WCS in exposure with a distortion model based on camera
928  geometry.
929 
930  Parameters
931  ----------
932  exposure : `lsst.afw.image.Exposure`
933  Exposure to process. Must contain a Detector and WCS. The
934  exposure is modified.
935  camera : `lsst.afw.cameraGeom.Camera`
936  Camera geometry.
937 
938  Raises
939  ------
940  RuntimeError
941  Raised if ``exposure`` is lacking a Detector or WCS, or if
942  ``camera`` is None.
943  Notes
944  -----
945  Add a model for optical distortion based on geometry found in ``camera``
946  and the ``exposure``'s detector. The raw input exposure is assumed
947  have a TAN WCS that has no compensation for optical distortion.
948  Two other possibilities are:
949  - The raw input exposure already has a model for optical distortion,
950  as is the case for raw DECam data.
951  In that case you should set config.doAddDistortionModel False.
952  - The raw input exposure has a model for distortion, but it has known
953  deficiencies severe enough to be worth fixing (e.g. because they
954  cause problems for fitting a better WCS). In that case you should
955  override this method with a version suitable for your raw data.
956 
957  """
958  wcs = exposure.getWcs()
959  if wcs is None:
960  raise RuntimeError("exposure has no WCS")
961  if camera is None:
962  raise RuntimeError("camera is None")
963  detector = exposure.getDetector()
964  if detector is None:
965  raise RuntimeError("exposure has no Detector")
966  pixelToFocalPlane = detector.getTransform(camGeom.PIXELS, camGeom.FOCAL_PLANE)
967  focalPlaneToFieldAngle = camera.getTransformMap().getTransform(camGeom.FOCAL_PLANE,
968  camGeom.FIELD_ANGLE)
969  distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle)
970  exposure.setWcs(distortedWcs)
971 
972 
973 def applyGains(exposure, normalizeGains=False):
974  """Scale an exposure by the amplifier gains.
975 
976  Parameters
977  ----------
978  exposure : `lsst.afw.image.Exposure`
979  Exposure to process. The image is modified.
980  normalizeGains : `Bool`, optional
981  If True, then amplifiers are scaled to force the median of
982  each amplifier to equal the median of those medians.
983  """
984  ccd = exposure.getDetector()
985  ccdImage = exposure.getMaskedImage()
986 
987  medians = []
988  for amp in ccd:
989  sim = ccdImage.Factory(ccdImage, amp.getBBox())
990  sim *= amp.getGain()
991 
992  if normalizeGains:
993  medians.append(numpy.median(sim.getImage().getArray()))
994 
995  if normalizeGains:
996  median = numpy.median(numpy.array(medians))
997  for index, amp in enumerate(ccd):
998  sim = ccdImage.Factory(ccdImage, amp.getDataSec())
999  sim *= median/medians[index]
1000 
1001 
1003  """Grow the saturation trails by an amount dependent on the width of the trail.
1004 
1005  Parameters
1006  ----------
1007  mask : `lsst.afw.image.Mask`
1008  Mask which will have the saturated areas grown.
1009  """
1010 
1011  extraGrowDict = {}
1012  for i in range(1, 6):
1013  extraGrowDict[i] = 0
1014  for i in range(6, 8):
1015  extraGrowDict[i] = 1
1016  for i in range(8, 10):
1017  extraGrowDict[i] = 3
1018  extraGrowMax = 4
1019 
1020  if extraGrowMax <= 0:
1021  return
1022 
1023  saturatedBit = mask.getPlaneBitMask("SAT")
1024 
1025  xmin, ymin = mask.getBBox().getMin()
1026  width = mask.getWidth()
1027 
1028  thresh = afwDetection.Threshold(saturatedBit, afwDetection.Threshold.BITMASK)
1029  fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
1030 
1031  for fp in fpList:
1032  for s in fp.getSpans():
1033  x0, x1 = s.getX0(), s.getX1()
1034 
1035  extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
1036  if extraGrow > 0:
1037  y = s.getY() - ymin
1038  x0 -= xmin + extraGrow
1039  x1 -= xmin - extraGrow
1040 
1041  if x0 < 0:
1042  x0 = 0
1043  if x1 >= width - 1:
1044  x1 = width - 1
1045 
1046  mask.array[y, x0:x1+1] |= saturatedBit
1047 
1048 
1049 def setBadRegions(exposure, badStatistic="MEDIAN"):
1050  """Set all BAD areas of the chip to the average of the rest of the exposure
1051 
1052  Parameters
1053  ----------
1054  exposure : `lsst.afw.image.Exposure`
1055  Exposure to mask. The exposure mask is modified.
1056  badStatistic : `str`, optional
1057  Statistic to use to generate the replacement value from the
1058  image data. Allowed values are 'MEDIAN' or 'MEANCLIP'.
1059 
1060  Returns
1061  -------
1062  badPixelCount : scalar
1063  Number of bad pixels masked.
1064  badPixelValue : scalar
1065  Value substituted for bad pixels.
1066 
1067  Raises
1068  ------
1069  RuntimeError
1070  Raised if `badStatistic` is not an allowed value.
1071  """
1072  if badStatistic == "MEDIAN":
1073  statistic = afwMath.MEDIAN
1074  elif badStatistic == "MEANCLIP":
1075  statistic = afwMath.MEANCLIP
1076  else:
1077  raise RuntimeError("Impossible method %s of bad region correction" % badStatistic)
1078 
1079  mi = exposure.getMaskedImage()
1080  mask = mi.getMask()
1081  BAD = mask.getPlaneBitMask("BAD")
1082  INTRP = mask.getPlaneBitMask("INTRP")
1083 
1084  sctrl = afwMath.StatisticsControl()
1085  sctrl.setAndMask(BAD)
1086  value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
1087 
1088  maskArray = mask.getArray()
1089  imageArray = mi.getImage().getArray()
1090  badPixels = numpy.logical_and((maskArray & BAD) > 0, (maskArray & INTRP) == 0)
1091  imageArray[:] = numpy.where(badPixels, value, imageArray)
1092 
1093  return badPixels.sum(), value
def illuminationCorrection(maskedImage, illumMaskedImage, illumScale)
def addDistortionModel(exposure, camera)
Update the WCS in exposure with a distortion model based on camera geometry.
def saturationCorrection(maskedImage, saturation, fwhm, growFootprints=1, interpolate=True, maskName='SAT', fallbackValue=None)
def setBadRegions(exposure, badStatistic="MEDIAN")
def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain)
def transposeDefectList(defectList)
def getDefectListFromMask(maskedImage, maskName)
def interpolateFromMask(maskedImage, fwhm, growSaturatedFootprints=1, maskNameList=['SAT'], fallbackValue=None)
def interpolateDefectList(maskedImage, defectList, fwhm, fallbackValue=None)
Definition: isrFunctions.py:77
def defectListFromFootprintList(fpList)
def transposeMaskedImage(maskedImage)
Definition: isrFunctions.py:57
def trimToMatchCalibBBox(rawMaskedImage, calibMaskedImage)
def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None)
def flatCorrection(maskedImage, flatMaskedImage, scalingType, userScale=1.0, invert=False, trimToFit=False)
def makeThresholdMask(maskedImage, threshold, growFootprints=1, maskName='SAT')
def applyGains(exposure, normalizeGains=False)
def darkCorrection(maskedImage, darkMaskedImage, expScale, darkScale, invert=False, trimToFit=False)
def biasCorrection(maskedImage, biasMaskedImage, trimToFit=False)
def gainContext(exp, image, apply)
def updateVariance(maskedImage, gain, readNoise)
def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1, collapseRej=3.0, statControl=None, overscanIsInt=True)
def maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD')