26 import lsst.pex.config
as pexConfig
37 from lsst.meas.algorithms import SourceDetectionTask, SingleGaussianPsf, ObjectSizeStarSelectorTask
38 from lsst.ip.diffim import (DipoleAnalysis, SourceFlagChecker, KernelCandidateF, makeKernelBasisList,
39 KernelCandidateQa, DiaCatalogSourceSelectorTask, DiaCatalogSourceSelectorConfig,
40 GetCoaddAsTemplateTask, GetCalexpAsTemplateTask, DipoleFitTask,
41 DecorrelateALKernelSpatialTask, subtractAlgorithmRegistry)
46 __all__ = [
"ImageDifferenceConfig",
"ImageDifferenceTask"]
47 FwhmPerSigma = 2*math.sqrt(2*math.log(2))
52 """Config for ImageDifferenceTask 54 doAddCalexpBackground = pexConfig.Field(dtype=bool, default=
False,
55 doc=
"Add background to calexp before processing it. " 56 "Useful as ipDiffim does background matching.")
57 doUseRegister = pexConfig.Field(dtype=bool, default=
True,
58 doc=
"Use image-to-image registration to align template with " 60 doDebugRegister = pexConfig.Field(dtype=bool, default=
False,
61 doc=
"Writing debugging data for doUseRegister")
62 doSelectSources = pexConfig.Field(dtype=bool, default=
True,
63 doc=
"Select stars to use for kernel fitting")
64 doSelectDcrCatalog = pexConfig.Field(dtype=bool, default=
False,
65 doc=
"Select stars of extreme color as part of the control sample")
66 doSelectVariableCatalog = pexConfig.Field(dtype=bool, default=
False,
67 doc=
"Select stars that are variable to be part " 68 "of the control sample")
69 doSubtract = pexConfig.Field(dtype=bool, default=
True, doc=
"Compute subtracted exposure?")
70 doPreConvolve = pexConfig.Field(dtype=bool, default=
True,
71 doc=
"Convolve science image by its PSF before PSF-matching?")
72 doScaleTemplateVariance = pexConfig.Field(dtype=bool, default=
False,
73 doc=
"Scale variance of the template before PSF matching")
74 useGaussianForPreConvolution = pexConfig.Field(dtype=bool, default=
True,
75 doc=
"Use a simple gaussian PSF model for pre-convolution " 76 "(else use fit PSF)? Ignored if doPreConvolve false.")
77 doDetection = pexConfig.Field(dtype=bool, default=
True, doc=
"Detect sources?")
78 doDecorrelation = pexConfig.Field(dtype=bool, default=
False,
79 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L " 80 "kernel convolution? If True, also update the diffim PSF.")
81 doMerge = pexConfig.Field(dtype=bool, default=
True,
82 doc=
"Merge positive and negative diaSources with grow radius " 83 "set by growFootprint")
84 doMatchSources = pexConfig.Field(dtype=bool, default=
True,
85 doc=
"Match diaSources with input calexp sources and ref catalog sources")
86 doMeasurement = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure diaSources?")
87 doDipoleFitting = pexConfig.Field(dtype=bool, default=
True, doc=
"Measure dipoles using new algorithm?")
88 doForcedMeasurement = pexConfig.Field(
91 doc=
"Force photometer diaSource locations on PVI?")
92 doWriteSubtractedExp = pexConfig.Field(dtype=bool, default=
True, doc=
"Write difference exposure?")
93 doWriteMatchedExp = pexConfig.Field(dtype=bool, default=
False,
94 doc=
"Write warped and PSF-matched template coadd exposure?")
95 doWriteSources = pexConfig.Field(dtype=bool, default=
True, doc=
"Write sources?")
96 doAddMetrics = pexConfig.Field(dtype=bool, default=
True,
97 doc=
"Add columns to the source table to hold analysis metrics?")
99 coaddName = pexConfig.Field(
100 doc=
"coadd name: typically one of deep, goodSeeing, or dcr",
104 convolveTemplate = pexConfig.Field(
105 doc=
"Which image gets convolved (default = template)",
109 refObjLoader = pexConfig.ConfigurableField(
110 target=LoadIndexedReferenceObjectsTask,
111 doc=
"reference object loader",
113 astrometer = pexConfig.ConfigurableField(
114 target=AstrometryTask,
115 doc=
"astrometry task; used to match sources to reference objects, but not to fit a WCS",
117 sourceSelector = pexConfig.ConfigurableField(
118 target=ObjectSizeStarSelectorTask,
119 doc=
"Source selection algorithm",
121 subtract = subtractAlgorithmRegistry.makeField(
"Subtraction Algorithm", default=
"al")
122 decorrelate = pexConfig.ConfigurableField(
123 target=DecorrelateALKernelSpatialTask,
124 doc=
"Decorrelate effects of A&L kernel convolution on image difference, only if doSubtract is True. " 125 "If this option is enabled, then detection.thresholdValue should be set to 5.0 (rather than the " 128 doSpatiallyVarying = pexConfig.Field(
131 doc=
"If using Zogy or A&L decorrelation, perform these on a grid across the " 132 "image in order to allow for spatial variations" 134 detection = pexConfig.ConfigurableField(
135 target=SourceDetectionTask,
136 doc=
"Low-threshold detection for final measurement",
138 measurement = pexConfig.ConfigurableField(
139 target=DipoleFitTask,
140 doc=
"Enable updated dipole fitting method",
142 forcedMeasurement = pexConfig.ConfigurableField(
143 target=ForcedMeasurementTask,
144 doc=
"Subtask to force photometer PVI at diaSource location.",
146 getTemplate = pexConfig.ConfigurableField(
147 target=GetCoaddAsTemplateTask,
148 doc=
"Subtask to retrieve template exposure and sources",
150 scaleVariance = pexConfig.ConfigurableField(
151 target=ScaleVarianceTask,
152 doc=
"Subtask to rescale the variance of the template " 153 "to the statistically expected level" 155 controlStepSize = pexConfig.Field(
156 doc=
"What step size (every Nth one) to select a control sample from the kernelSources",
160 controlRandomSeed = pexConfig.Field(
161 doc=
"Random seed for shuffing the control sample",
165 register = pexConfig.ConfigurableField(
167 doc=
"Task to enable image-to-image image registration (warping)",
169 kernelSourcesFromRef = pexConfig.Field(
170 doc=
"Select sources to measure kernel from reference catalog if True, template if false",
174 templateSipOrder = pexConfig.Field(
175 dtype=int, default=2,
176 doc=
"Sip Order for fitting the Template Wcs (default is too high, overfitting)" 178 growFootprint = pexConfig.Field(
179 dtype=int, default=2,
180 doc=
"Grow positive and negative footprints by this amount before merging" 182 diaSourceMatchRadius = pexConfig.Field(
183 dtype=float, default=0.5,
184 doc=
"Match radius (in arcseconds) for DiaSource to Source association" 190 self.
subtract[
'al'].kernel.name =
"AL" 191 self.
subtract[
'al'].kernel.active.fitForBackground =
True 192 self.
subtract[
'al'].kernel.active.spatialKernelOrder = 1
193 self.
subtract[
'al'].kernel.active.spatialBgOrder = 2
200 self.
detection.thresholdPolarity =
"both" 202 self.
detection.reEstimateBackground =
False 203 self.
detection.thresholdType =
"pixel_stdev" 209 self.
measurement.algorithms.names.add(
'base_PeakLikelihoodFlux')
213 "id":
"objectId",
"parent":
"parentObjectId",
"coord_ra":
"coord_ra",
"coord_dec":
"coord_dec"}
221 pexConfig.Config.validate(self)
223 raise ValueError(
"Subtraction must be enabled for kernel metrics calculation.")
225 raise ValueError(
"Either doSubtract or doDetection must be enabled.")
227 raise ValueError(
"Kernel metrics does not exist in zogy subtraction.")
229 raise ValueError(
"Cannot run source measurement without source detection.")
231 raise ValueError(
"Cannot run source merging without source detection.")
233 raise ValueError(
"doUseRegister=True and doSelectSources=False. " 234 "Cannot run RegisterTask without selecting sources.")
236 raise ValueError(
"doPreConvolve=True and doDecorrelation=True and " 237 "convolveTemplate=False is not supported.")
240 raise ValueError(
"Mis-matched coaddName and getTemplate.coaddName in the config.")
247 return pipeBase.TaskRunner.getTargetList(parsedCmd, templateIdList=parsedCmd.templateId.idList,
252 """Subtract an image from a template and measure the result 254 ConfigClass = ImageDifferenceConfig
255 RunnerClass = ImageDifferenceTaskRunner
256 _DefaultName =
"imageDifference" 259 """!Construct an ImageDifference Task 261 @param[in] butler Butler object to use in constructing reference object loaders 263 pipeBase.CmdLineTask.__init__(self, **kwargs)
264 self.makeSubtask(
"getTemplate")
266 self.makeSubtask(
"subtract")
268 if self.config.subtract.name ==
'al' and self.config.doDecorrelation:
269 self.makeSubtask(
"decorrelate")
271 if self.config.doScaleTemplateVariance:
272 self.makeSubtask(
"scaleVariance")
274 if self.config.doUseRegister:
275 self.makeSubtask(
"register")
276 self.
schema = afwTable.SourceTable.makeMinimalSchema()
278 if self.config.doSelectSources:
279 self.makeSubtask(
"sourceSelector")
280 if self.config.kernelSourcesFromRef:
281 self.makeSubtask(
'refObjLoader', butler=butler)
282 self.makeSubtask(
"astrometer", refObjLoader=self.refObjLoader)
285 if self.config.doDetection:
286 self.makeSubtask(
"detection", schema=self.
schema)
287 if self.config.doMeasurement:
288 self.makeSubtask(
"measurement", schema=self.
schema,
290 if self.config.doForcedMeasurement:
292 "ip_diffim_forced_PsfFlux_instFlux",
"D",
293 "Forced PSF flux measured on the direct image.")
295 "ip_diffim_forced_PsfFlux_instFluxErr",
"D",
296 "Forced PSF flux error measured on the direct image.")
298 "ip_diffim_forced_PsfFlux_area",
"F",
299 "Forced PSF flux effective area of PSF.",
302 "ip_diffim_forced_PsfFlux_flag",
"Flag",
303 "Forced PSF flux general failure flag.")
305 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
"Flag",
306 "Forced PSF flux not enough non-rejected pixels in data to attempt the fit.")
308 "ip_diffim_forced_PsfFlux_flag_edge",
"Flag",
309 "Forced PSF flux object was too close to the edge of the image to use the full PSF model.")
310 self.makeSubtask(
"forcedMeasurement", refSchema=self.
schema)
311 if self.config.doMatchSources:
312 self.
schema.addField(
"refMatchId",
"L",
"unique id of reference catalog match")
313 self.
schema.addField(
"srcMatchId",
"L",
"unique id of source match")
317 """Create IdFactory instance for unique 64 bit diaSource id-s. 325 Number of used bits in ``expId``. 329 The diasource id-s consists of the ``expId`` stored fixed in the highest value 330 ``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the 331 low value end of the integer. 335 idFactory: `lsst.afw.table.IdFactory` 337 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
341 """Subtract an image from a template coadd and measure the result. 343 Data I/O wrapper around `run` using the butler in Gen2. 347 sensorRef : `lsst.daf.persistence.ButlerDataRef` 348 Sensor-level butler data reference, used for the following data products: 355 - self.config.coaddName + "Coadd_skyMap" 356 - self.config.coaddName + "Coadd" 357 Input or output, depending on config: 358 - self.config.coaddName + "Diff_subtractedExp" 359 Output, depending on config: 360 - self.config.coaddName + "Diff_matchedExp" 361 - self.config.coaddName + "Diff_src" 365 results : `lsst.pipe.base.Struct` 366 Returns the Struct by `run`. 368 subtractedExposureName = self.config.coaddName +
"Diff_differenceExp" 369 subtractedExposure =
None 371 calexpBackgroundExposure =
None 372 self.log.info(
"Processing %s" % (sensorRef.dataId))
377 idFactory = self.
makeIdFactory(expId=int(sensorRef.get(
"ccdExposureId")),
378 expBits=sensorRef.get(
"ccdExposureId_bits"))
379 if self.config.doAddCalexpBackground:
380 calexpBackgroundExposure = sensorRef.get(
"calexpBackground")
383 exposure = sensorRef.get(
"calexp", immediate=
True)
386 template = self.getTemplate.
run(exposure, sensorRef, templateIdList=templateIdList)
388 if sensorRef.datasetExists(
"src"):
389 self.log.info(
"Source selection via src product")
391 selectSources = sensorRef.get(
"src")
393 if not self.config.doSubtract
and self.config.doDetection:
395 subtractedExposure = sensorRef.get(subtractedExposureName)
398 results = self.
run(exposure=exposure,
399 selectSources=selectSources,
400 templateExposure=template.exposure,
401 templateSources=template.sources,
403 calexpBackgroundExposure=calexpBackgroundExposure,
404 subtractedExposure=subtractedExposure)
406 if self.config.doWriteSources
and results.diaSources
is not None:
407 sensorRef.put(results.diaSources, self.config.coaddName +
"Diff_diaSrc")
408 if self.config.doWriteMatchedExp:
409 sensorRef.put(results.matchedExposure, self.config.coaddName +
"Diff_matchedExp")
410 if self.config.doAddMetrics
and self.config.doSelectSources:
411 sensorRef.put(results.selectSources, self.config.coaddName +
"Diff_kernelSrc")
412 if self.config.doWriteSubtractedExp:
413 sensorRef.put(results.subtractedExposure, subtractedExposureName)
416 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None,
417 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None):
418 """PSF matches, subtract two images and perform detection on the difference image. 422 exposure : `lsst.afw.image.ExposureF`, optional 423 The science exposure, the minuend in the image subtraction. 424 Can be None only if ``config.doSubtract==False``. 425 selectSources : `lsst.afw.table.SourceCatalog`, optional 426 Identified sources on the science exposure. This catalog is used to 427 select sources in order to perform the AL PSF matching on stamp images 428 around them. The selection steps depend on config options and whether 429 ``templateSources`` and ``matchingSources`` specified. 430 templateExposure : `lsst.afw.image.ExposureF`, optional 431 The template to be subtracted from ``exposure`` in the image subtraction. 432 The template exposure should cover the same sky area as the science exposure. 433 It is either a stich of patches of a coadd skymap image or a calexp 434 of the same pointing as the science exposure. Can be None only 435 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None. 436 templateSources : `lsst.afw.table.SourceCatalog`, optional 437 Identified sources on the template exposure. 438 idFactory : `lsst.afw.table.IdFactory` 439 Generator object to assign ids to detected sources in the difference image. 440 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional 441 Background exposure to be added back to the science exposure 442 if ``config.doAddCalexpBackground==True`` 443 subtractedExposure : `lsst.afw.image.ExposureF`, optional 444 If ``config.doSubtract==False`` and ``config.doDetection==True``, 445 performs the post subtraction source detection only on this exposure. 446 Otherwise should be None. 450 results : `lsst.pipe.base.Struct` 451 ``subtractedExposure`` : `lsst.afw.image.ExposureF` 453 ``matchedExposure`` : `lsst.afw.image.ExposureF` 454 The matched PSF exposure. 455 ``subtractRes`` : `lsst.pipe.base.Struct` 456 The returned result structure of the ImagePsfMatchTask subtask. 457 ``diaSources`` : `lsst.afw.table.SourceCatalog` 458 The catalog of detected sources. 459 ``selectSources`` : `lsst.afw.table.SourceCatalog` 460 The input source catalog with optionally added Qa information. 464 The following major steps are included: 466 - warp template coadd to match WCS of image 467 - PSF match image to warped template 468 - subtract image from PSF-matched, warped template 472 For details about the image subtraction configuration modes 473 see `lsst.ip.diffim`. 476 controlSources =
None 480 if self.config.doAddCalexpBackground:
481 mi = exposure.getMaskedImage()
482 mi += calexpBackgroundExposure.getImage()
484 if not exposure.hasPsf():
485 raise pipeBase.TaskError(
"Exposure has no psf")
486 sciencePsf = exposure.getPsf()
488 if self.config.doSubtract:
489 if self.config.doScaleTemplateVariance:
490 templateVarFactor = self.scaleVariance.
run(
491 templateExposure.getMaskedImage())
492 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
494 if self.config.subtract.name ==
'zogy':
495 subtractRes = self.subtract.subtractExposures(templateExposure, exposure,
497 spatiallyVarying=self.config.doSpatiallyVarying,
498 doPreConvolve=self.config.doPreConvolve)
499 subtractedExposure = subtractRes.subtractedExposure
501 elif self.config.subtract.name ==
'al':
503 scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
504 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
512 if self.config.doPreConvolve:
513 convControl = afwMath.ConvolutionControl()
515 srcMI = exposure.getMaskedImage()
516 destMI = srcMI.Factory(srcMI.getDimensions())
518 if self.config.useGaussianForPreConvolution:
520 kWidth, kHeight = sciencePsf.getLocalKernel().getDimensions()
525 afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(), convControl)
526 exposure.setMaskedImage(destMI)
527 scienceSigmaPost = scienceSigmaOrig*math.sqrt(2)
529 scienceSigmaPost = scienceSigmaOrig
534 if self.config.doSelectSources:
535 if selectSources
is None:
536 self.log.warn(
"Src product does not exist; running detection, measurement, selection")
538 selectSources = self.subtract.getSelectSources(
540 sigma=scienceSigmaPost,
541 doSmooth=
not self.doPreConvolve,
545 if self.config.doAddMetrics:
548 nparam = len(makeKernelBasisList(self.subtract.config.kernel.active,
549 referenceFwhmPix=scienceSigmaPost*FwhmPerSigma,
550 targetFwhmPix=templateSigma*FwhmPerSigma))
557 kcQa = KernelCandidateQa(nparam)
558 selectSources = kcQa.addToSchema(selectSources)
559 if self.config.kernelSourcesFromRef:
561 astromRet = self.astrometer.loadAndMatch(exposure=exposure, sourceCat=selectSources)
562 matches = astromRet.matches
563 elif templateSources:
565 mc = afwTable.MatchControl()
566 mc.findOnlyClosest =
False 567 matches = afwTable.matchRaDec(templateSources, selectSources, 1.0*geom.arcseconds,
570 raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False," 571 "but template sources not available. Cannot match science " 572 "sources with template sources. Run process* on data from " 573 "which templates are built.")
575 kernelSources = self.sourceSelector.
run(selectSources, exposure=exposure,
576 matches=matches).sourceCat
577 random.shuffle(kernelSources, random.random)
578 controlSources = kernelSources[::self.config.controlStepSize]
579 kernelSources = [k
for i, k
in enumerate(kernelSources)
580 if i % self.config.controlStepSize]
582 if self.config.doSelectDcrCatalog:
583 redSelector = DiaCatalogSourceSelectorTask(
584 DiaCatalogSourceSelectorConfig(grMin=self.sourceSelector.config.grMax,
586 redSources = redSelector.selectStars(exposure, selectSources, matches=matches).starCat
587 controlSources.extend(redSources)
589 blueSelector = DiaCatalogSourceSelectorTask(
590 DiaCatalogSourceSelectorConfig(grMin=-99.999,
591 grMax=self.sourceSelector.config.grMin))
592 blueSources = blueSelector.selectStars(exposure, selectSources,
593 matches=matches).starCat
594 controlSources.extend(blueSources)
596 if self.config.doSelectVariableCatalog:
597 varSelector = DiaCatalogSourceSelectorTask(
598 DiaCatalogSourceSelectorConfig(includeVariable=
True))
599 varSources = varSelector.selectStars(exposure, selectSources, matches=matches).starCat
600 controlSources.extend(varSources)
602 self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)" 603 % (len(kernelSources), len(selectSources), len(controlSources)))
607 if self.config.doUseRegister:
608 self.log.info(
"Registering images")
610 if templateSources
is None:
614 templateSigma = templateExposure.getPsf().computeShape().getDeterminantRadius()
615 templateSources = self.subtract.getSelectSources(
624 wcsResults = self.
fitAstrometry(templateSources, templateExposure, selectSources)
625 warpedExp = self.register.warpExposure(templateExposure, wcsResults.wcs,
626 exposure.getWcs(), exposure.getBBox())
627 templateExposure = warpedExp
632 if self.config.doDebugRegister:
634 srcToMatch = {x.second.getId(): x.first
for x
in matches}
636 refCoordKey = wcsResults.matches[0].first.getTable().getCoordKey()
637 inCentroidKey = wcsResults.matches[0].second.getTable().getCentroidKey()
638 sids = [m.first.getId()
for m
in wcsResults.matches]
639 positions = [m.first.get(refCoordKey)
for m
in wcsResults.matches]
640 residuals = [m.first.get(refCoordKey).getOffsetFrom(wcsResults.wcs.pixelToSky(
641 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
642 allresids = dict(zip(sids, zip(positions, residuals)))
644 cresiduals = [m.first.get(refCoordKey).getTangentPlaneOffset(
645 wcsResults.wcs.pixelToSky(
646 m.second.get(inCentroidKey)))
for m
in wcsResults.matches]
647 colors = numpy.array([-2.5*numpy.log10(srcToMatch[x].get(
"g")) +
648 2.5*numpy.log10(srcToMatch[x].get(
"r")) 649 for x
in sids
if x
in srcToMatch.keys()])
650 dlong = numpy.array([r[0].asArcseconds()
for s, r
in zip(sids, cresiduals)
651 if s
in srcToMatch.keys()])
652 dlat = numpy.array([r[1].asArcseconds()
for s, r
in zip(sids, cresiduals)
653 if s
in srcToMatch.keys()])
654 idx1 = numpy.where(colors < self.sourceSelector.config.grMin)
655 idx2 = numpy.where((colors >= self.sourceSelector.config.grMin) &
656 (colors <= self.sourceSelector.config.grMax))
657 idx3 = numpy.where(colors > self.sourceSelector.config.grMax)
658 rms1Long = IqrToSigma*(
659 (numpy.percentile(dlong[idx1], 75) - numpy.percentile(dlong[idx1], 25)))
660 rms1Lat = IqrToSigma*(numpy.percentile(dlat[idx1], 75) -
661 numpy.percentile(dlat[idx1], 25))
662 rms2Long = IqrToSigma*(
663 (numpy.percentile(dlong[idx2], 75) - numpy.percentile(dlong[idx2], 25)))
664 rms2Lat = IqrToSigma*(numpy.percentile(dlat[idx2], 75) -
665 numpy.percentile(dlat[idx2], 25))
666 rms3Long = IqrToSigma*(
667 (numpy.percentile(dlong[idx3], 75) - numpy.percentile(dlong[idx3], 25)))
668 rms3Lat = IqrToSigma*(numpy.percentile(dlat[idx3], 75) -
669 numpy.percentile(dlat[idx3], 25))
670 self.log.info(
"Blue star offsets'': %.3f %.3f, %.3f %.3f" %
671 (numpy.median(dlong[idx1]), rms1Long,
672 numpy.median(dlat[idx1]), rms1Lat))
673 self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
674 (numpy.median(dlong[idx2]), rms2Long,
675 numpy.median(dlat[idx2]), rms2Lat))
676 self.log.info(
"Red star offsets'': %.3f %.3f, %.3f %.3f" %
677 (numpy.median(dlong[idx3]), rms3Long,
678 numpy.median(dlat[idx3]), rms3Lat))
680 self.metadata.add(
"RegisterBlueLongOffsetMedian", numpy.median(dlong[idx1]))
681 self.metadata.add(
"RegisterGreenLongOffsetMedian", numpy.median(dlong[idx2]))
682 self.metadata.add(
"RegisterRedLongOffsetMedian", numpy.median(dlong[idx3]))
683 self.metadata.add(
"RegisterBlueLongOffsetStd", rms1Long)
684 self.metadata.add(
"RegisterGreenLongOffsetStd", rms2Long)
685 self.metadata.add(
"RegisterRedLongOffsetStd", rms3Long)
687 self.metadata.add(
"RegisterBlueLatOffsetMedian", numpy.median(dlat[idx1]))
688 self.metadata.add(
"RegisterGreenLatOffsetMedian", numpy.median(dlat[idx2]))
689 self.metadata.add(
"RegisterRedLatOffsetMedian", numpy.median(dlat[idx3]))
690 self.metadata.add(
"RegisterBlueLatOffsetStd", rms1Lat)
691 self.metadata.add(
"RegisterGreenLatOffsetStd", rms2Lat)
692 self.metadata.add(
"RegisterRedLatOffsetStd", rms3Lat)
699 self.log.info(
"Subtracting images")
700 subtractRes = self.subtract.subtractExposures(
701 templateExposure=templateExposure,
702 scienceExposure=exposure,
703 candidateList=kernelSources,
704 convolveTemplate=self.config.convolveTemplate,
705 doWarping=
not self.config.doUseRegister
707 subtractedExposure = subtractRes.subtractedExposure
709 if self.config.doDetection:
710 self.log.info(
"Computing diffim PSF")
713 if not subtractedExposure.hasPsf():
714 if self.config.convolveTemplate:
715 subtractedExposure.setPsf(exposure.getPsf())
717 subtractedExposure.setPsf(templateExposure.getPsf())
724 if self.config.doDecorrelation
and self.config.doSubtract:
726 if preConvPsf
is not None:
727 preConvKernel = preConvPsf.getLocalKernel()
728 if self.config.convolveTemplate:
729 self.log.info(
"Decorrelation after template image convolution")
730 decorrResult = self.decorrelate.
run(exposure, templateExposure,
732 subtractRes.psfMatchingKernel,
733 spatiallyVarying=self.config.doSpatiallyVarying,
734 preConvKernel=preConvKernel)
736 self.log.info(
"Decorrelation after science image convolution")
737 decorrResult = self.decorrelate.
run(templateExposure, exposure,
739 subtractRes.psfMatchingKernel,
740 spatiallyVarying=self.config.doSpatiallyVarying,
741 preConvKernel=preConvKernel)
742 subtractedExposure = decorrResult.correctedExposure
746 if self.config.doDetection:
747 self.log.info(
"Running diaSource detection")
749 mask = subtractedExposure.getMaskedImage().getMask()
750 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
752 table = afwTable.SourceTable.make(self.
schema, idFactory)
754 results = self.detection.makeSourceCatalog(
756 exposure=subtractedExposure,
757 doSmooth=
not self.config.doPreConvolve
760 if self.config.doMerge:
761 fpSet = results.fpSets.positive
762 fpSet.merge(results.fpSets.negative, self.config.growFootprint,
763 self.config.growFootprint,
False)
764 diaSources = afwTable.SourceCatalog(table)
765 fpSet.makeSources(diaSources)
766 self.log.info(
"Merging detections into %d sources" % (len(diaSources)))
768 diaSources = results.sources
770 if self.config.doMeasurement:
771 newDipoleFitting = self.config.doDipoleFitting
772 self.log.info(
"Running diaSource measurement: newDipoleFitting=%r", newDipoleFitting)
773 if not newDipoleFitting:
775 self.measurement.
run(diaSources, subtractedExposure)
778 if self.config.doSubtract
and 'matchedExposure' in subtractRes.getDict():
779 self.measurement.
run(diaSources, subtractedExposure, exposure,
780 subtractRes.matchedExposure)
782 self.measurement.
run(diaSources, subtractedExposure, exposure)
784 if self.config.doForcedMeasurement:
787 forcedSources = self.forcedMeasurement.generateMeasCat(
788 exposure, diaSources, subtractedExposure.getWcs())
789 self.forcedMeasurement.
run(forcedSources, exposure, diaSources, subtractedExposure.getWcs())
790 mapper = afwTable.SchemaMapper(forcedSources.schema, diaSources.schema)
791 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFlux")[0],
792 "ip_diffim_forced_PsfFlux_instFlux",
True)
793 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_instFluxErr")[0],
794 "ip_diffim_forced_PsfFlux_instFluxErr",
True)
795 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_area")[0],
796 "ip_diffim_forced_PsfFlux_area",
True)
797 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag")[0],
798 "ip_diffim_forced_PsfFlux_flag",
True)
799 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_noGoodPixels")[0],
800 "ip_diffim_forced_PsfFlux_flag_noGoodPixels",
True)
801 mapper.addMapping(forcedSources.schema.find(
"base_PsfFlux_flag_edge")[0],
802 "ip_diffim_forced_PsfFlux_flag_edge",
True)
803 for diaSource, forcedSource
in zip(diaSources, forcedSources):
804 diaSource.assign(forcedSource, mapper)
807 if self.config.doMatchSources:
808 if selectSources
is not None:
810 matchRadAsec = self.config.diaSourceMatchRadius
811 matchRadPixel = matchRadAsec/exposure.getWcs().getPixelScale().asArcseconds()
813 srcMatches = afwTable.matchXy(selectSources, diaSources, matchRadPixel)
814 srcMatchDict = dict([(srcMatch.second.getId(), srcMatch.first.getId())
for 815 srcMatch
in srcMatches])
816 self.log.info(
"Matched %d / %d diaSources to sources" % (len(srcMatchDict),
819 self.log.warn(
"Src product does not exist; cannot match with diaSources")
823 refAstromConfig = AstrometryConfig()
824 refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
825 refAstrometer = AstrometryTask(refAstromConfig)
826 astromRet = refAstrometer.run(exposure=exposure, sourceCat=diaSources)
827 refMatches = astromRet.matches
828 if refMatches
is None:
829 self.log.warn(
"No diaSource matches with reference catalog")
832 self.log.info(
"Matched %d / %d diaSources to reference catalog" % (len(refMatches),
834 refMatchDict = dict([(refMatch.second.getId(), refMatch.first.getId())
for 835 refMatch
in refMatches])
838 for diaSource
in diaSources:
839 sid = diaSource.getId()
840 if sid
in srcMatchDict:
841 diaSource.set(
"srcMatchId", srcMatchDict[sid])
842 if sid
in refMatchDict:
843 diaSource.set(
"refMatchId", refMatchDict[sid])
845 if self.config.doAddMetrics
and self.config.doSelectSources:
846 self.log.info(
"Evaluating metrics and control sample")
849 for cell
in subtractRes.kernelCellSet.getCellList():
850 for cand
in cell.begin(
False):
851 kernelCandList.append(cand)
854 basisList = kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelList()
855 nparam = len(kernelCandList[0].getKernel(KernelCandidateF.ORIG).getKernelParameters())
858 diffimTools.sourceTableToCandidateList(controlSources,
859 subtractRes.warpedExposure, exposure,
860 self.config.subtract.kernel.active,
861 self.config.subtract.kernel.active.detectionConfig,
862 self.log, doBuild=
True, basisList=basisList))
864 KernelCandidateQa.apply(kernelCandList, subtractRes.psfMatchingKernel,
865 subtractRes.backgroundModel, dof=nparam)
866 KernelCandidateQa.apply(controlCandList, subtractRes.psfMatchingKernel,
867 subtractRes.backgroundModel)
869 if self.config.doDetection:
870 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids, diaSources)
872 KernelCandidateQa.aggregate(selectSources, self.metadata, allresids)
874 self.
runDebug(exposure, subtractRes, selectSources, kernelSources, diaSources)
875 return pipeBase.Struct(
876 subtractedExposure=subtractedExposure,
877 matchedExposure=subtractRes.matchedExposure,
878 subtractRes=subtractRes,
879 diaSources=diaSources,
880 selectSources=selectSources
884 """Fit the relative astrometry between templateSources and selectSources 889 Remove this method. It originally fit a new WCS to the template before calling register.run 890 because our TAN-SIP fitter behaved badly for points far from CRPIX, but that's been fixed. 891 It remains because a subtask overrides it. 893 results = self.register.
run(templateSources, templateExposure.getWcs(),
894 templateExposure.getBBox(), selectSources)
897 def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
898 """Make debug plots and displays. 902 Test and update for current debug display and slot names 912 disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
913 if not maskTransparency:
915 disp.setMaskTransparency(maskTransparency)
917 if display
and showSubtracted:
918 disp.mtv(subtractRes.subtractedExposure, title=
"Subtracted image")
919 mi = subtractRes.subtractedExposure.getMaskedImage()
920 x0, y0 = mi.getX0(), mi.getY0()
921 with disp.Buffering():
923 x, y = s.getX() - x0, s.getY() - y0
924 ctype =
"red" if s.get(
"flags_negative")
else "yellow" 925 if (s.get(
"base_PixelFlags_flag_interpolatedCenter")
or 926 s.get(
"base_PixelFlags_flag_saturatedCenter")
or 927 s.get(
"base_PixelFlags_flag_crCenter")):
929 elif (s.get(
"base_PixelFlags_flag_interpolated")
or 930 s.get(
"base_PixelFlags_flag_saturated")
or 931 s.get(
"base_PixelFlags_flag_cr")):
935 disp.dot(ptype, x, y, size=4, ctype=ctype)
938 if display
and showPixelResiduals
and selectSources:
939 nonKernelSources = []
940 for source
in selectSources:
941 if source
not in kernelSources:
942 nonKernelSources.append(source)
944 diUtils.plotPixelResiduals(exposure,
945 subtractRes.warpedExposure,
946 subtractRes.subtractedExposure,
947 subtractRes.kernelCellSet,
948 subtractRes.psfMatchingKernel,
949 subtractRes.backgroundModel,
951 self.subtract.config.kernel.active.detectionConfig,
953 diUtils.plotPixelResiduals(exposure,
954 subtractRes.warpedExposure,
955 subtractRes.subtractedExposure,
956 subtractRes.kernelCellSet,
957 subtractRes.psfMatchingKernel,
958 subtractRes.backgroundModel,
960 self.subtract.config.kernel.active.detectionConfig,
962 if display
and showDiaSources:
963 flagChecker = SourceFlagChecker(diaSources)
964 isFlagged = [flagChecker(x)
for x
in diaSources]
965 isDipole = [x.get(
"ip_diffim_ClassificationDipole_value")
for x
in diaSources]
966 diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
967 frame=lsstDebug.frame)
970 if display
and showDipoles:
971 DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
972 frame=lsstDebug.frame)
975 def _getConfigName(self):
976 """Return the name of the config dataset 978 return "%sDiff_config" % (self.config.coaddName,)
980 def _getMetadataName(self):
981 """Return the name of the metadata dataset 983 return "%sDiff_metadata" % (self.config.coaddName,)
986 """Return a dict of empty catalogs for each catalog dataset produced by this task.""" 987 diaSrc = afwTable.SourceCatalog(self.
schema)
989 return {self.config.coaddName +
"Diff_diaSrc": diaSrc}
992 def _makeArgumentParser(cls):
993 """Create an argument parser 996 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=12345 ccd=1,2")
997 parser.add_id_argument(
"--templateId",
"calexp", doMakeDataRefList=
True,
998 help=
"Template data ID in case of calexp template," 999 " e.g. --templateId visit=6789")
1004 winter2013WcsShift = pexConfig.Field(dtype=float, default=0.0,
1005 doc=
"Shift stars going into RegisterTask by this amount")
1006 winter2013WcsRms = pexConfig.Field(dtype=float, default=0.0,
1007 doc=
"Perturb stars going into RegisterTask by this amount")
1010 ImageDifferenceConfig.setDefaults(self)
1011 self.
getTemplate.retarget(GetCalexpAsTemplateTask)
1015 """!Image difference Task used in the Winter 2013 data challege. 1016 Enables testing the effects of registration shifts and scatter. 1018 For use with winter 2013 simulated images: 1019 Use --templateId visit=88868666 for sparse data 1020 --templateId visit=22222200 for dense data (g) 1021 --templateId visit=11111100 for dense data (i) 1023 ConfigClass = Winter2013ImageDifferenceConfig
1024 _DefaultName =
"winter2013ImageDifference" 1027 ImageDifferenceTask.__init__(self, **kwargs)
1030 """Fit the relative astrometry between templateSources and selectSources""" 1031 if self.config.winter2013WcsShift > 0.0:
1033 self.config.winter2013WcsShift)
1034 cKey = templateSources[0].getTable().getCentroidKey()
1035 for source
in templateSources:
1036 centroid = source.get(cKey)
1037 source.set(cKey, centroid + offset)
1038 elif self.config.winter2013WcsRms > 0.0:
1039 cKey = templateSources[0].getTable().getCentroidKey()
1040 for source
in templateSources:
1041 offset =
geom.Extent2D(self.config.winter2013WcsRms*numpy.random.normal(),
1042 self.config.winter2013WcsRms*numpy.random.normal())
1043 centroid = source.get(cKey)
1044 source.set(cKey, centroid + offset)
1046 results = self.register.
run(templateSources, templateExposure.getWcs(),
1047 templateExposure.getBBox(), selectSources)
def __init__(self, butler=None, kwargs)
Construct an ImageDifference Task.
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def runDataRef(self, sensorRef, templateIdList=None)
def __init__(self, kwargs)
def getSchemaCatalogs(self)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources)
Image difference Task used in the Winter 2013 data challege.
def makeIdFactory(expId, expBits)
def fitAstrometry(self, templateSources, templateExposure, selectSources)
def getTargetList(parsedCmd, kwargs)
def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None, idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None)