37 from .coaddBase
import CoaddBaseTask, SelectDataIdContainer, makeSkyInfo
38 from .interpImage
import InterpImageTask
39 from .scaleZeroPoint
import ScaleZeroPointTask
40 from .coaddHelpers
import groupPatchExposures, getGroupDataRef
41 from .scaleVariance
import ScaleVarianceTask
45 __all__ = [
"AssembleCoaddTask",
"AssembleCoaddConfig",
"SafeClipAssembleCoaddTask",
46 "SafeClipAssembleCoaddConfig",
"CompareWarpAssembleCoaddTask",
"CompareWarpAssembleCoaddConfig"]
50 """Configuration parameters for the `AssembleCoaddTask`. 54 The `doMaskBrightObjects` and `brightObjectMaskName` configuration options 55 only set the bitplane config.brightObjectMaskName. To make this useful you 56 *must* also configure the flags.pixel algorithm, for example by adding 60 config.measurement.plugins["base_PixelFlags"].masksFpCenter.append("BRIGHT_OBJECT") 61 config.measurement.plugins["base_PixelFlags"].masksFpAnywhere.append("BRIGHT_OBJECT") 63 to your measureCoaddSources.py and forcedPhotCoadd.py config overrides. 65 warpType = pexConfig.Field(
66 doc=
"Warp name: one of 'direct' or 'psfMatched'",
70 subregionSize = pexConfig.ListField(
72 doc=
"Width, height of stack subregion size; " 73 "make small enough that a full stack of images will fit into memory at once.",
77 statistic = pexConfig.Field(
79 doc=
"Main stacking statistic for aggregating over the epochs.",
82 doSigmaClip = pexConfig.Field(
84 doc=
"Perform sigma clipped outlier rejection with MEANCLIP statistic? (DEPRECATED)",
87 sigmaClip = pexConfig.Field(
89 doc=
"Sigma for outlier rejection; ignored if non-clipping statistic selected.",
92 clipIter = pexConfig.Field(
94 doc=
"Number of iterations of outlier rejection; ignored if non-clipping statistic selected.",
97 calcErrorFromInputVariance = pexConfig.Field(
99 doc=
"Calculate coadd variance from input variance by stacking statistic." 100 "Passed to StatisticsControl.setCalcErrorFromInputVariance()",
103 scaleZeroPoint = pexConfig.ConfigurableField(
104 target=ScaleZeroPointTask,
105 doc=
"Task to adjust the photometric zero point of the coadd temp exposures",
107 doInterp = pexConfig.Field(
108 doc=
"Interpolate over NaN pixels? Also extrapolate, if necessary, but the results are ugly.",
112 interpImage = pexConfig.ConfigurableField(
113 target=InterpImageTask,
114 doc=
"Task to interpolate (and extrapolate) over NaN pixels",
116 doWrite = pexConfig.Field(
117 doc=
"Persist coadd?",
121 doNImage = pexConfig.Field(
122 doc=
"Create image of number of contributing exposures for each pixel",
126 doUsePsfMatchedPolygons = pexConfig.Field(
127 doc=
"Use ValidPolygons from shrunk Psf-Matched Calexps? Should be set to True by CompareWarp only.",
131 maskPropagationThresholds = pexConfig.DictField(
134 doc=(
"Threshold (in fractional weight) of rejection at which we propagate a mask plane to " 135 "the coadd; that is, we set the mask bit on the coadd if the fraction the rejected frames " 136 "would have contributed exceeds this value."),
137 default={
"SAT": 0.1},
139 removeMaskPlanes = pexConfig.ListField(dtype=str, default=[
"NOT_DEBLENDED"],
140 doc=
"Mask planes to remove before coadding")
141 doMaskBrightObjects = pexConfig.Field(dtype=bool, default=
False,
142 doc=
"Set mask and flag bits for bright objects?")
143 brightObjectMaskName = pexConfig.Field(dtype=str, default=
"BRIGHT_OBJECT",
144 doc=
"Name of mask bit used for bright objects")
145 coaddPsf = pexConfig.ConfigField(
146 doc=
"Configuration for CoaddPsf",
147 dtype=measAlg.CoaddPsfConfig,
149 doAttachTransmissionCurve = pexConfig.Field(
150 dtype=bool, default=
False, optional=
False,
151 doc=(
"Attach a piecewise TransmissionCurve for the coadd? " 152 "(requires all input Exposures to have TransmissionCurves).")
154 inputWarps = pipeBase.InputDatasetField(
155 doc=(
"Input list of warps to be assemebled i.e. stacked." 156 "WarpType (e.g. direct, psfMatched) is controlled by we warpType config parameter"),
157 nameTemplate=
"{inputCoaddName}Coadd_{warpType}Warp",
158 storageClass=
"ExposureF",
159 dimensions=(
"Tract",
"Patch",
"SkyMap",
"Visit",
"Instrument"),
162 skyMap = pipeBase.InputDatasetField(
163 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
164 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
165 storageClass=
"SkyMap",
166 dimensions=(
"SkyMap", ),
169 brightObjectMask = pipeBase.InputDatasetField(
170 doc=(
"Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane" 172 name=
"brightObjectMask",
173 storageClass=
"ObjectMaskCatalog",
174 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter"),
177 coaddExposure = pipeBase.OutputDatasetField(
178 doc=
"Output coadded exposure, produced by stacking input warps",
179 nameTemplate=
"{outputCoaddName}Coadd",
180 storageClass=
"ExposureF",
181 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter"),
184 nImage = pipeBase.OutputDatasetField(
185 doc=
"Output image of number of input images per pixel",
186 nameTemplate=
"{outputCoaddName}Coadd_nImage",
187 storageClass=
"ImageU",
188 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter"),
195 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep",
197 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
204 log.warn(
"Config doPsfMatch deprecated. Setting warpType='psfMatched'")
207 log.warn(
'doSigmaClip deprecated. To replicate behavior, setting statistic to "MEANCLIP"')
209 if self.
doInterp and self.
statistic not in [
'MEAN',
'MEDIAN',
'MEANCLIP',
'VARIANCE',
'VARIANCECLIP']:
210 raise ValueError(
"Must set doInterp=False for statistic=%s, which does not " 211 "compute and set a non-zero coadd variance estimate." % (self.
statistic))
213 unstackableStats = [
'NOTHING',
'ERROR',
'ORMASK']
214 if not hasattr(afwMath.Property, self.
statistic)
or self.
statistic in unstackableStats:
215 stackableStats = [str(k)
for k
in afwMath.Property.__members__.keys()
216 if str(k)
not in unstackableStats]
217 raise ValueError(
"statistic %s is not allowed. Please choose one of %s." 222 """Assemble a coadded image from a set of warps (coadded temporary exposures). 224 We want to assemble a coadded image from a set of Warps (also called 225 coadded temporary exposures or ``coaddTempExps``). 226 Each input Warp covers a patch on the sky and corresponds to a single 227 run/visit/exposure of the covered patch. We provide the task with a list 228 of Warps (``selectDataList``) from which it selects Warps that cover the 229 specified patch (pointed at by ``dataRef``). 230 Each Warp that goes into a coadd will typically have an independent 231 photometric zero-point. Therefore, we must scale each Warp to set it to 232 a common photometric zeropoint. WarpType may be one of 'direct' or 233 'psfMatched', and the boolean configs `config.makeDirect` and 234 `config.makePsfMatched` set which of the warp types will be coadded. 235 The coadd is computed as a mean with optional outlier rejection. 236 Criteria for outlier rejection are set in `AssembleCoaddConfig`. 237 Finally, Warps can have bad 'NaN' pixels which received no input from the 238 source calExps. We interpolate over these bad (NaN) pixels. 240 `AssembleCoaddTask` uses several sub-tasks. These are 242 - `ScaleZeroPointTask` 243 - create and use an ``imageScaler`` object to scale the photometric zeropoint for each Warp 245 - interpolate across bad pixels (NaN) in the final coadd 247 You can retarget these subtasks if you wish. 251 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 252 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 253 `baseDebug` for more about ``debug.py`` files. `AssembleCoaddTask` has 254 no debug variables of its own. Some of the subtasks may support debug 255 variables. See the documentation for the subtasks for further information. 259 `AssembleCoaddTask` assembles a set of warped images into a coadded image. 260 The `AssembleCoaddTask` can be invoked by running ``assembleCoadd.py`` 261 with the flag '--legacyCoadd'. Usage of assembleCoadd.py expects two 262 inputs: a data reference to the tract patch and filter to be coadded, and 263 a list of Warps to attempt to coadd. These are specified using ``--id`` and 264 ``--selectId``, respectively: 268 --id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 269 --selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]] 271 Only the Warps that cover the specified tract and patch will be coadded. 272 A list of the available optional arguments can be obtained by calling 273 ``assembleCoadd.py`` with the ``--help`` command line argument: 277 assembleCoadd.py --help 279 To demonstrate usage of the `AssembleCoaddTask` in the larger context of 280 multi-band processing, we will generate the HSC-I & -R band coadds from 281 HSC engineering test data provided in the ``ci_hsc`` package. To begin, 282 assuming that the lsst stack has been already set up, we must set up the 283 obs_subaru and ``ci_hsc`` packages. This defines the environment variable 284 ``$CI_HSC_DIR`` and points at the location of the package. The raw HSC 285 data live in the ``$CI_HSC_DIR/raw directory``. To begin assembling the 286 coadds, we must first 289 - process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 291 - create a skymap that covers the area of the sky present in the raw exposures 293 - warp the individual calibrated exposures to the tangent plane of the coadd 295 We can perform all of these steps by running 299 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 301 This will produce warped exposures for each visit. To coadd the warped 302 data, we call assembleCoadd.py as follows: 306 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 307 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 308 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 309 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 310 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 311 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 312 --selectId visit=903988 ccd=24 314 that will process the HSC-I band data. The results are written in 315 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 317 You may also choose to run: 321 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 322 assembleCoadd.py --legacyCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R \ 323 --selectId visit=903334 ccd=16 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 \ 324 --selectId visit=903334 ccd=100 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 \ 325 --selectId visit=903338 ccd=18 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 \ 326 --selectId visit=903342 ccd=10 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 \ 327 --selectId visit=903344 ccd=5 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 \ 328 --selectId visit=903346 ccd=6 --selectId visit=903346 ccd=12 330 to generate the coadd for the HSC-R band if you are interested in 331 following multiBand Coadd processing as discussed in `pipeTasks_multiBand` 332 (but note that normally, one would use the `SafeClipAssembleCoaddTask` 333 rather than `AssembleCoaddTask` to make the coadd. 335 ConfigClass = AssembleCoaddConfig
336 _DefaultName =
"assembleCoadd" 341 argNames = [
"config",
"name",
"parentTask",
"log"]
342 kwargs.update({k: v
for k, v
in zip(argNames, args)})
343 warnings.warn(
"AssembleCoadd received positional args, and casting them as kwargs: %s. " 344 "PipelineTask will not take positional args" % argNames, FutureWarning)
347 self.makeSubtask(
"interpImage")
348 self.makeSubtask(
"scaleZeroPoint")
350 if self.config.doMaskBrightObjects:
351 mask = afwImage.Mask()
354 except pexExceptions.LsstCppException:
355 raise RuntimeError(
"Unable to define mask plane for bright objects; planes used are %s" %
356 mask.getMaskPlaneDict().keys())
363 """Return output dataset type descriptors 365 Remove output dataset types not produced by the Task 368 if not config.doNImage:
369 outputTypeDict.pop(
"nImage",
None)
370 return outputTypeDict
374 """Return input dataset type descriptors 376 Remove input dataset types not used by the Task 379 if not config.doMaskBrightObjects:
380 inputTypeDict.pop(
"brightObjectMask",
None)
385 return frozenset([
"brightObjectMask"])
388 """Assemble a coadd from a set of Warps. 390 PipelineTask (Gen3) entry point to Coadd a set of Warps. 391 Analogous to `runDataRef`, it prepares all the data products to be 392 passed to `run`, and processes the results before returning to struct 393 of results to be written out. AssembleCoadd cannot fit all Warps in memory. 394 Therefore, its inputs are accessed subregion by subregion 395 by the `lsst.daf.butler.ShimButler` that quacks like a Gen2 396 `lsst.daf.persistence.Butler`. Updates to this method should 397 correspond to an update in `runDataRef` while both entry points 403 Keys are the names of the configs describing input dataset types. 404 Values are input Python-domain data objects (or lists of objects) 405 retrieved from data butler. 406 inputDataIds : `dict` 407 Keys are the names of the configs describing input dataset types. 408 Values are DataIds (or lists of DataIds) that task consumes for 409 corresponding dataset type. 410 outputDataIds : `dict` 411 Keys are the names of the configs describing input dataset types. 412 Values are DataIds (or lists of DataIds) that task is to produce 413 for corresponding dataset type. 414 butler : `lsst.daf.butler.Butler` 415 Gen3 Butler object for fetching additional data products before 420 result : `lsst.pipe.base.Struct` 421 Result struct with components: 423 - ``coaddExposure`` : coadded exposure (``lsst.afw.image.Exposure``) 424 - ``nImage``: N Image (``lsst.afw.image.Image``) 428 skyMap = inputData[
"skyMap"]
429 outputDataId = next(iter(outputDataIds.values()))
431 tractId=outputDataId[
'tract'],
432 patchId=outputDataId[
'patch'])
436 warpRefList = [butlerShim.dataRef(self.config.inputWarps.name, dataId=dataId)
437 for dataId
in inputDataIds[
'inputWarps']]
440 patchRef = butlerShim.dataRef(self.config.coaddExposure.name, dataId=outputDataIds[
'coaddExposure'])
444 self.log.info(
"Found %d %s", len(inputs.tempExpRefList),
446 if len(inputs.tempExpRefList) == 0:
447 self.log.warn(
"No coadd temporary exposures found")
452 retStruct = self.
run(inputData[
'skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
453 inputs.weightList, supplementaryData=supplementaryData)
459 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
460 """Assemble a coadd from a set of Warps. 462 Pipebase.CmdlineTask entry point to Coadd a set of Warps. 463 Compute weights to be applied to each Warp and 464 find scalings to match the photometric zeropoint to a reference Warp. 465 Assemble the Warps using `run`. Interpolate over NaNs and 466 optionally write the coadd to disk. Return the coadded exposure. 470 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 471 Data reference defining the patch for coaddition and the 472 reference Warp (if ``config.autoReference=False``). 473 Used to access the following data products: 474 - ``self.config.coaddName + "Coadd_skyMap"`` 475 - ``self.config.coaddName + "Coadd_ + <warpType> + "Warp"`` (optionally) 476 - ``self.config.coaddName + "Coadd"`` 477 selectDataList : `list` 478 List of data references to Calexps. Data to be coadded will be 479 selected from this list based on overlap with the patch defined 480 by dataRef, grouped by visit, and converted to a list of data 483 List of data references to Warps to be coadded. 484 Note: `warpRefList` is just the new name for `tempExpRefList`. 488 retStruct : `lsst.pipe.base.Struct` 489 Result struct with components: 491 - ``coaddExposure``: coadded exposure (``Exposure``). 492 - ``nImage``: exposure count image (``Image``). 494 if selectDataList
and warpRefList:
495 raise RuntimeError(
"runDataRef received both a selectDataList and warpRefList, " 496 "and which to use is ambiguous. Please pass only one.")
499 if warpRefList
is None:
500 calExpRefList = self.
selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
501 if len(calExpRefList) == 0:
502 self.log.warn(
"No exposures to coadd")
504 self.log.info(
"Coadding %d exposures", len(calExpRefList))
509 self.log.info(
"Found %d %s", len(inputData.tempExpRefList),
511 if len(inputData.tempExpRefList) == 0:
512 self.log.warn(
"No coadd temporary exposures found")
517 retStruct = self.
run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
518 inputData.weightList, supplementaryData=supplementaryData)
521 if self.config.doWrite:
524 if self.config.doNImage
and retStruct.nImage
is not None:
530 """Interpolate over missing data and mask bright stars. 534 coaddExposure : `lsst.afw.image.Exposure` 535 The coadded exposure to process. 536 dataRef : `lsst.daf.persistence.ButlerDataRef` 537 Butler data reference for supplementary data. 539 if self.config.doInterp:
540 self.interpImage.
run(coaddExposure.getMaskedImage(), planeName=
"NO_DATA")
542 varArray = coaddExposure.variance.array
543 with numpy.errstate(invalid=
"ignore"):
544 varArray[:] = numpy.where(varArray > 0, varArray, numpy.inf)
546 if self.config.doMaskBrightObjects:
551 """Make additional inputs to run() specific to subclasses (Gen2) 553 Duplicates interface of `runDataRef` method 554 Available to be implemented by subclasses only if they need the 555 coadd dataRef for performing preliminary processing before 556 assembling the coadd. 560 dataRef : `lsst.daf.persistence.ButlerDataRef` 561 Butler data reference for supplementary data. 562 selectDataList : `list` 563 List of data references to Warps. 568 """Make additional inputs to run() specific to subclasses (Gen3) 570 Duplicates interface of`adaptArgsAndRun` method. 571 Available to be implemented by subclasses only if they need the 572 coadd dataRef for performing preliminary processing before 573 assembling the coadd. 578 Keys are the names of the configs describing input dataset types. 579 Values are input Python-domain data objects (or lists of objects) 580 retrieved from data butler. 581 inputDataIds : `dict` 582 Keys are the names of the configs describing input dataset types. 583 Values are DataIds (or lists of DataIds) that task consumes for 584 corresponding dataset type. 585 DataIds are guaranteed to match data objects in ``inputData``. 586 outputDataIds : `dict` 587 Keys are the names of the configs describing input dataset types. 588 Values are DataIds (or lists of DataIds) that task is to produce 589 for corresponding dataset type. 590 butler : `lsst.daf.butler.Butler` 591 Gen3 Butler object for fetching additional data products before 596 result : `lsst.pipe.base.Struct` 597 Contains whatever additional data the subclass's `run` method needs 602 """Generate list data references corresponding to warped exposures 603 that lie within the patch to be coadded. 608 Data reference for patch. 609 calExpRefList : `list` 610 List of data references for input calexps. 614 tempExpRefList : `list` 615 List of Warp/CoaddTempExp data references. 617 butler = patchRef.getButler()
618 groupData =
groupPatchExposures(patchRef, calExpRefList, self.getCoaddDatasetName(self.warpType),
619 self.getTempExpDatasetName(self.warpType))
620 tempExpRefList = [
getGroupDataRef(butler, self.getTempExpDatasetName(self.warpType),
621 g, groupData.keys)
for 622 g
in groupData.groups.keys()]
623 return tempExpRefList
626 """Prepare the input warps for coaddition by measuring the weight for 627 each warp and the scaling for the photometric zero point. 629 Each Warp has its own photometric zeropoint and background variance. 630 Before coadding these Warps together, compute a scale factor to 631 normalize the photometric zeropoint and compute the weight for each Warp. 636 List of data references to tempExp 640 result : `lsst.pipe.base.Struct` 641 Result struct with components: 643 - ``tempExprefList``: `list` of data references to tempExp. 644 - ``weightList``: `list` of weightings. 645 - ``imageScalerList``: `list` of image scalers. 647 statsCtrl = afwMath.StatisticsControl()
648 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
649 statsCtrl.setNumIter(self.config.clipIter)
651 statsCtrl.setNanSafe(
True)
659 for tempExpRef
in refList:
660 if not tempExpRef.datasetExists(tempExpName):
661 self.log.warn(
"Could not find %s %s; skipping it", tempExpName, tempExpRef.dataId)
664 tempExp = tempExpRef.get(tempExpName, immediate=
True)
666 if numpy.isnan(tempExp.image.array).all():
668 maskedImage = tempExp.getMaskedImage()
669 imageScaler = self.scaleZeroPoint.computeImageScaler(
674 imageScaler.scaleMaskedImage(maskedImage)
675 except Exception
as e:
676 self.log.warn(
"Scaling failed for %s (skipping it): %s", tempExpRef.dataId, e)
678 statObj = afwMath.makeStatistics(maskedImage.getVariance(), maskedImage.getMask(),
679 afwMath.MEANCLIP, statsCtrl)
680 meanVar, meanVarErr = statObj.getResult(afwMath.MEANCLIP)
681 weight = 1.0 / float(meanVar)
682 if not numpy.isfinite(weight):
683 self.log.warn(
"Non-finite weight for %s: skipping", tempExpRef.dataId)
685 self.log.info(
"Weight of %s %s = %0.3f", tempExpName, tempExpRef.dataId, weight)
690 tempExpRefList.append(tempExpRef)
691 weightList.append(weight)
692 imageScalerList.append(imageScaler)
694 return pipeBase.Struct(tempExpRefList=tempExpRefList, weightList=weightList,
695 imageScalerList=imageScalerList)
698 """Prepare the statistics for coadding images. 702 mask : `int`, optional 703 Bit mask value to exclude from coaddition. 707 stats : `lsst.pipe.base.Struct` 708 Statistics structure with the following fields: 710 - ``statsCtrl``: Statistics control object for coadd 711 (`lsst.afw.math.StatisticsControl`) 712 - ``statsFlags``: Statistic for coadd (`lsst.afw.math.Property`) 716 statsCtrl = afwMath.StatisticsControl()
717 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
718 statsCtrl.setNumIter(self.config.clipIter)
719 statsCtrl.setAndMask(mask)
720 statsCtrl.setNanSafe(
True)
721 statsCtrl.setWeighted(
True)
722 statsCtrl.setCalcErrorFromInputVariance(self.config.calcErrorFromInputVariance)
723 for plane, threshold
in self.config.maskPropagationThresholds.items():
724 bit = afwImage.Mask.getMaskPlane(plane)
725 statsCtrl.setMaskPropagationThreshold(bit, threshold)
726 statsFlags = afwMath.stringToStatisticsProperty(self.config.statistic)
727 return pipeBase.Struct(ctrl=statsCtrl, flags=statsFlags)
729 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
730 altMaskList=None, mask=None, supplementaryData=None):
731 """Assemble a coadd from input warps 733 Assemble the coadd using the provided list of coaddTempExps. Since 734 the full coadd covers a patch (a large area), the assembly is 735 performed over small areas on the image at a time in order to 736 conserve memory usage. Iterate over subregions within the outer 737 bbox of the patch using `assembleSubregion` to stack the corresponding 738 subregions from the coaddTempExps with the statistic specified. 739 Set the edge bits the coadd mask based on the weight map. 743 skyInfo : `lsst.pipe.base.Struct` 744 Struct with geometric information about the patch. 745 tempExpRefList : `list` 746 List of data references to Warps (previously called CoaddTempExps). 747 imageScalerList : `list` 748 List of image scalers. 751 altMaskList : `list`, optional 752 List of alternate masks to use rather than those stored with 754 mask : `int`, optional 755 Bit mask value to exclude from coaddition. 756 supplementaryData : lsst.pipe.base.Struct, optional 757 Struct with additional data products needed to assemble coadd. 758 Only used by subclasses that implement `makeSupplementaryData` 763 result : `lsst.pipe.base.Struct` 764 Result struct with components: 766 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 767 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 770 self.log.info(
"Assembling %s %s", len(tempExpRefList), tempExpName)
773 if altMaskList
is None:
774 altMaskList = [
None]*len(tempExpRefList)
776 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
777 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
778 coaddExposure.getInfo().setCoaddInputs(self.inputRecorder.makeCoaddInputs())
780 coaddMaskedImage = coaddExposure.getMaskedImage()
781 subregionSizeArr = self.config.subregionSize
782 subregionSize = afwGeom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
784 if self.config.doNImage:
785 nImage = afwImage.ImageU(skyInfo.bbox)
788 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
791 weightList, altMaskList, stats.flags, stats.ctrl,
793 except Exception
as e:
794 self.log.fatal(
"Cannot compute coadd %s: %s", subBBox, e)
799 coaddUtils.setCoaddEdgeBits(coaddMaskedImage.getMask(), coaddMaskedImage.getVariance())
800 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage)
803 """Set the metadata for the coadd. 805 This basic implementation sets the filter from the first input. 809 coaddExposure : `lsst.afw.image.Exposure` 810 The target exposure for the coadd. 811 tempExpRefList : `list` 812 List of data references to tempExp. 816 assert len(tempExpRefList) == len(weightList),
"Length mismatch" 821 tempExpList = [tempExpRef.get(tempExpName +
"_sub",
822 bbox=afwGeom.Box2I(coaddExposure.getBBox().getMin(),
823 afwGeom.Extent2I(1, 1)), immediate=
True)
824 for tempExpRef
in tempExpRefList]
825 numCcds = sum(len(tempExp.getInfo().getCoaddInputs().ccds)
for tempExp
in tempExpList)
827 coaddExposure.setFilter(tempExpList[0].getFilter())
828 coaddInputs = coaddExposure.getInfo().getCoaddInputs()
829 coaddInputs.ccds.reserve(numCcds)
830 coaddInputs.visits.reserve(len(tempExpList))
832 for tempExp, weight
in zip(tempExpList, weightList):
833 self.inputRecorder.addVisitToCoadd(coaddInputs, tempExp, weight)
835 if self.config.doUsePsfMatchedPolygons:
838 coaddInputs.visits.sort()
844 modelPsfList = [tempExp.getPsf()
for tempExp
in tempExpList]
845 modelPsfWidthList = [modelPsf.computeBBox().getWidth()
for modelPsf
in modelPsfList]
846 psf = modelPsfList[modelPsfWidthList.index(max(modelPsfWidthList))]
848 psf = measAlg.CoaddPsf(coaddInputs.ccds, coaddExposure.getWcs(),
849 self.config.coaddPsf.makeControl())
850 coaddExposure.setPsf(psf)
851 apCorrMap = measAlg.makeCoaddApCorrMap(coaddInputs.ccds, coaddExposure.getBBox(afwImage.PARENT),
852 coaddExposure.getWcs())
853 coaddExposure.getInfo().setApCorrMap(apCorrMap)
854 if self.config.doAttachTransmissionCurve:
855 transmissionCurve = measAlg.makeCoaddTransmissionCurve(coaddExposure.getWcs(), coaddInputs.ccds)
856 coaddExposure.getInfo().setTransmissionCurve(transmissionCurve)
858 def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList,
859 altMaskList, statsFlags, statsCtrl, nImage=None):
860 """Assemble the coadd for a sub-region. 862 For each coaddTempExp, check for (and swap in) an alternative mask 863 if one is passed. Remove mask planes listed in 864 `config.removeMaskPlanes`. Finally, stack the actual exposures using 865 `lsst.afw.math.statisticsStack` with the statistic specified by 866 statsFlags. Typically, the statsFlag will be one of lsst.afw.math.MEAN for 867 a mean-stack or `lsst.afw.math.MEANCLIP` for outlier rejection using 868 an N-sigma clipped mean where N and iterations are specified by 869 statsCtrl. Assign the stacked subregion back to the coadd. 873 coaddExposure : `lsst.afw.image.Exposure` 874 The target exposure for the coadd. 875 bbox : `lsst.afw.geom.Box` 877 tempExpRefList : `list` 878 List of data reference to tempExp. 879 imageScalerList : `list` 880 List of image scalers. 884 List of alternate masks to use rather than those stored with 885 tempExp, or None. Each element is dict with keys = mask plane 886 name to which to add the spans. 887 statsFlags : `lsst.afw.math.Property` 888 Property object for statistic for coadd. 889 statsCtrl : `lsst.afw.math.StatisticsControl` 890 Statistics control object for coadd. 891 nImage : `lsst.afw.image.ImageU`, optional 892 Keeps track of exposure count for each pixel. 894 self.log.debug(
"Computing coadd over %s", bbox)
896 coaddExposure.mask.addMaskPlane(
"REJECTED")
897 coaddExposure.mask.addMaskPlane(
"CLIPPED")
898 coaddExposure.mask.addMaskPlane(
"SENSOR_EDGE")
900 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
902 if nImage
is not None:
903 subNImage = afwImage.ImageU(bbox.getWidth(), bbox.getHeight())
904 for tempExpRef, imageScaler, altMask
in zip(tempExpRefList, imageScalerList, altMaskList):
905 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
906 maskedImage = exposure.getMaskedImage()
907 mask = maskedImage.getMask()
908 if altMask
is not None:
910 imageScaler.scaleMaskedImage(maskedImage)
914 if nImage
is not None:
915 subNImage.getArray()[maskedImage.getMask().getArray() & statsCtrl.getAndMask() == 0] += 1
916 if self.config.removeMaskPlanes:
918 maskedImageList.append(maskedImage)
920 with self.timer(
"stack"):
921 coaddSubregion = afwMath.statisticsStack(maskedImageList, statsFlags, statsCtrl, weightList,
924 coaddExposure.maskedImage.assign(coaddSubregion, bbox)
925 if nImage
is not None:
926 nImage.assign(subNImage, bbox)
929 """Unset the mask of an image for mask planes specified in the config. 933 maskedImage : `lsst.afw.image.MaskedImage` 934 The masked image to be modified. 936 mask = maskedImage.getMask()
937 for maskPlane
in self.config.removeMaskPlanes:
939 mask &= ~mask.getPlaneBitMask(maskPlane)
940 except Exception
as e:
941 self.log.warn(
"Unable to remove mask plane %s: %s", maskPlane, e.args[0])
945 """Map certain mask planes of the warps to new planes for the coadd. 947 If a pixel is rejected due to a mask value other than EDGE, NO_DATA, 948 or CLIPPED, set it to REJECTED on the coadd. 949 If a pixel is rejected due to EDGE, set the coadd pixel to SENSOR_EDGE. 950 If a pixel is rejected due to CLIPPED, set the coadd pixel to CLIPPED. 954 statsCtrl : `lsst.afw.math.StatisticsControl` 955 Statistics control object for coadd 959 maskMap : `list` of `tuple` of `int` 960 A list of mappings of mask planes of the warped exposures to 961 mask planes of the coadd. 963 edge = afwImage.Mask.getPlaneBitMask(
"EDGE")
964 noData = afwImage.Mask.getPlaneBitMask(
"NO_DATA")
965 clipped = afwImage.Mask.getPlaneBitMask(
"CLIPPED")
966 toReject = statsCtrl.getAndMask() & (~noData) & (~edge) & (~clipped)
967 maskMap = [(toReject, afwImage.Mask.getPlaneBitMask(
"REJECTED")),
968 (edge, afwImage.Mask.getPlaneBitMask(
"SENSOR_EDGE")),
973 """Apply in place alt mask formatted as SpanSets to a mask. 977 mask : `lsst.afw.image.Mask` 979 altMaskSpans : `dict` 980 SpanSet lists to apply. Each element contains the new mask 981 plane name (e.g. "CLIPPED and/or "NO_DATA") as the key, 982 and list of SpanSets to apply to the mask. 986 mask : `lsst.afw.image.Mask` 989 if self.config.doUsePsfMatchedPolygons:
990 if (
"NO_DATA" in altMaskSpans)
and (
"NO_DATA" in self.config.badMaskPlanes):
995 for spanSet
in altMaskSpans[
'NO_DATA']:
996 spanSet.clippedTo(mask.getBBox()).clearMask(mask, self.
getBadPixelMask())
998 for plane, spanSetList
in altMaskSpans.items():
999 maskClipValue = mask.addMaskPlane(plane)
1000 for spanSet
in spanSetList:
1001 spanSet.clippedTo(mask.getBBox()).setMask(mask, 2**maskClipValue)
1005 """Shrink coaddInputs' ccds' ValidPolygons in place. 1007 Either modify each ccd's validPolygon in place, or if CoaddInputs 1008 does not have a validPolygon, create one from its bbox. 1012 coaddInputs : `lsst.afw.image.coaddInputs` 1016 for ccd
in coaddInputs.ccds:
1017 polyOrig = ccd.getValidPolygon()
1018 validPolyBBox = polyOrig.getBBox()
if polyOrig
else ccd.getBBox()
1019 validPolyBBox.grow(-self.config.matchingKernelSize//2)
1021 validPolygon = polyOrig.intersectionSingle(validPolyBBox)
1023 validPolygon = afwGeom.polygon.Polygon(afwGeom.Box2D(validPolyBBox))
1024 ccd.setValidPolygon(validPolygon)
1027 """Retrieve the bright object masks. 1029 Returns None on failure. 1033 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1038 result : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 1039 Bright object mask from the Butler object, or None if it cannot 1043 return dataRef.get(
"brightObjectMask", immediate=
True)
1044 except Exception
as e:
1045 self.log.warn(
"Unable to read brightObjectMask for %s: %s", dataRef.dataId, e)
1049 """Set the bright object masks. 1053 exposure : `lsst.afw.image.Exposure` 1054 Exposure under consideration. 1055 dataId : `lsst.daf.persistence.dataId` 1056 Data identifier dict for patch. 1057 brightObjectMasks : `lsst.afw.table` 1058 Table of bright objects to mask. 1061 if brightObjectMasks
is None:
1062 self.log.warn(
"Unable to apply bright object mask: none supplied")
1064 self.log.info(
"Applying %d bright object masks to %s", len(brightObjectMasks), dataId)
1065 mask = exposure.getMaskedImage().getMask()
1066 wcs = exposure.getWcs()
1067 plateScale = wcs.getPixelScale().asArcseconds()
1069 for rec
in brightObjectMasks:
1070 center = afwGeom.PointI(wcs.skyToPixel(rec.getCoord()))
1071 if rec[
"type"] ==
"box":
1072 assert rec[
"angle"] == 0.0, (
"Angle != 0 for mask object %s" % rec[
"id"])
1073 width = rec[
"width"].asArcseconds()/plateScale
1074 height = rec[
"height"].asArcseconds()/plateScale
1076 halfSize = afwGeom.ExtentI(0.5*width, 0.5*height)
1077 bbox = afwGeom.Box2I(center - halfSize, center + halfSize)
1079 bbox = afwGeom.BoxI(afwGeom.PointI(int(center[0] - 0.5*width), int(center[1] - 0.5*height)),
1080 afwGeom.PointI(int(center[0] + 0.5*width), int(center[1] + 0.5*height)))
1081 spans = afwGeom.SpanSet(bbox)
1082 elif rec[
"type"] ==
"circle":
1083 radius = int(rec[
"radius"].asArcseconds()/plateScale)
1084 spans = afwGeom.SpanSet.fromShape(radius, offset=center)
1086 self.log.warn(
"Unexpected region type %s at %s" % rec[
"type"], center)
1091 """Set INEXACT_PSF mask plane. 1093 If any of the input images isn't represented in the coadd (due to 1094 clipped pixels or chip gaps), the `CoaddPsf` will be inexact. Flag 1099 mask : `lsst.afw.image.Mask` 1100 Coadded exposure's mask, modified in-place. 1102 mask.addMaskPlane(
"INEXACT_PSF")
1103 inexactPsf = mask.getPlaneBitMask(
"INEXACT_PSF")
1104 sensorEdge = mask.getPlaneBitMask(
"SENSOR_EDGE")
1105 clipped = mask.getPlaneBitMask(
"CLIPPED")
1106 rejected = mask.getPlaneBitMask(
"REJECTED")
1107 array = mask.getArray()
1108 selected = array & (sensorEdge | clipped | rejected) > 0
1109 array[selected] |= inexactPsf
1112 def _makeArgumentParser(cls):
1113 """Create an argument parser. 1115 parser = pipeBase.ArgumentParser(name=cls.
_DefaultName)
1116 parser.add_id_argument(
"--id", cls.
ConfigClass().coaddName +
"Coadd_" +
1118 help=
"data ID, e.g. --id tract=12345 patch=1,2",
1119 ContainerClass=AssembleCoaddDataIdContainer)
1120 parser.add_id_argument(
"--selectId",
"calexp", help=
"data ID, e.g. --selectId visit=6789 ccd=0..9",
1121 ContainerClass=SelectDataIdContainer)
1125 def _subBBoxIter(bbox, subregionSize):
1126 """Iterate over subregions of a bbox. 1130 bbox : `lsst.afw.geom.Box2I` 1131 Bounding box over which to iterate. 1132 subregionSize: `lsst.afw.geom.Extent2I` 1137 subBBox : `lsst.afw.geom.Box2I` 1138 Next sub-bounding box of size ``subregionSize`` or smaller; each ``subBBox`` 1139 is contained within ``bbox``, so it may be smaller than ``subregionSize`` at 1140 the edges of ``bbox``, but it will never be empty. 1143 raise RuntimeError(
"bbox %s is empty" % (bbox,))
1144 if subregionSize[0] < 1
or subregionSize[1] < 1:
1145 raise RuntimeError(
"subregionSize %s must be nonzero" % (subregionSize,))
1147 for rowShift
in range(0, bbox.getHeight(), subregionSize[1]):
1148 for colShift
in range(0, bbox.getWidth(), subregionSize[0]):
1149 subBBox = afwGeom.Box2I(bbox.getMin() + afwGeom.Extent2I(colShift, rowShift), subregionSize)
1151 if subBBox.isEmpty():
1152 raise RuntimeError(
"Bug: empty bbox! bbox=%s, subregionSize=%s, " 1153 "colShift=%s, rowShift=%s" %
1154 (bbox, subregionSize, colShift, rowShift))
1159 """A version of `lsst.pipe.base.DataIdContainer` specialized for assembleCoadd. 1163 """Make self.refList from self.idList. 1168 Results of parsing command-line (with ``butler`` and ``log`` elements). 1170 datasetType = namespace.config.coaddName +
"Coadd" 1171 keysCoadd = namespace.butler.getKeys(datasetType=datasetType, level=self.level)
1173 for dataId
in self.idList:
1175 for key
in keysCoadd:
1176 if key
not in dataId:
1177 raise RuntimeError(
"--id must include " + key)
1179 dataRef = namespace.butler.dataRef(
1180 datasetType=datasetType,
1183 self.refList.append(dataRef)
1187 """Function to count the number of pixels with a specific mask in a 1190 Find the intersection of mask & footprint. Count all pixels in the mask 1191 that are in the intersection that have bitmask set but do not have 1192 ignoreMask set. Return the count. 1196 mask : `lsst.afw.image.Mask` 1197 Mask to define intersection region by. 1198 footprint : `lsst.afw.detection.Footprint` 1199 Footprint to define the intersection region by. 1201 Specific mask that we wish to count the number of occurances of. 1203 Pixels to not consider. 1208 Count of number of pixels in footprint with specified mask. 1210 bbox = footprint.getBBox()
1211 bbox.clip(mask.getBBox(afwImage.PARENT))
1212 fp = afwImage.Mask(bbox)
1213 subMask = mask.Factory(mask, bbox, afwImage.PARENT)
1214 footprint.spans.setMask(fp, bitmask)
1215 return numpy.logical_and((subMask.getArray() & fp.getArray()) > 0,
1216 (subMask.getArray() & ignoreMask) == 0).sum()
1220 """Configuration parameters for the SafeClipAssembleCoaddTask. 1222 clipDetection = pexConfig.ConfigurableField(
1223 target=SourceDetectionTask,
1224 doc=
"Detect sources on difference between unclipped and clipped coadd")
1225 minClipFootOverlap = pexConfig.Field(
1226 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be clipped",
1230 minClipFootOverlapSingle = pexConfig.Field(
1231 doc=
"Minimum fractional overlap of clipped footprint with visit DETECTED to be " 1232 "clipped when only one visit overlaps",
1236 minClipFootOverlapDouble = pexConfig.Field(
1237 doc=
"Minimum fractional overlap of clipped footprints with visit DETECTED to be " 1238 "clipped when two visits overlap",
1242 maxClipFootOverlapDouble = pexConfig.Field(
1243 doc=
"Maximum fractional overlap of clipped footprints with visit DETECTED when " 1244 "considering two visits",
1248 minBigOverlap = pexConfig.Field(
1249 doc=
"Minimum number of pixels in footprint to use DETECTED mask from the single visits " 1250 "when labeling clipped footprints",
1256 """Set default values for clipDetection. 1260 The numeric values for these configuration parameters were 1261 empirically determined, future work may further refine them. 1263 AssembleCoaddConfig.setDefaults(self)
1279 log.warn(
"Additional Sigma-clipping not allowed in Safe-clipped Coadds. " 1280 "Ignoring doSigmaClip.")
1283 raise ValueError(
"Only MEAN statistic allowed for final stacking in SafeClipAssembleCoadd " 1284 "(%s chosen). Please set statistic to MEAN." 1286 AssembleCoaddTask.ConfigClass.validate(self)
1290 """Assemble a coadded image from a set of coadded temporary exposures, 1291 being careful to clip & flag areas with potential artifacts. 1293 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1294 we clip outliers). The problem with doing this is that when computing the 1295 coadd PSF at a given location, individual visit PSFs from visits with 1296 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1297 In this task, we correct for this behavior by creating a new 1298 ``badMaskPlane`` 'CLIPPED'. We populate this plane on the input 1299 coaddTempExps and the final coadd where 1301 i. difference imaging suggests that there is an outlier and 1302 ii. this outlier appears on only one or two images. 1304 Such regions will not contribute to the final coadd. Furthermore, any 1305 routine to determine the coadd PSF can now be cognizant of clipped regions. 1306 Note that the algorithm implemented by this task is preliminary and works 1307 correctly for HSC data. Parameter modifications and or considerable 1308 redesigning of the algorithm is likley required for other surveys. 1310 ``SafeClipAssembleCoaddTask`` uses a ``SourceDetectionTask`` 1311 "clipDetection" subtask and also sub-classes ``AssembleCoaddTask``. 1312 You can retarget the ``SourceDetectionTask`` "clipDetection" subtask 1317 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1318 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; 1319 see `baseDebug` for more about ``debug.py`` files. 1320 `SafeClipAssembleCoaddTask` has no debug variables of its own. 1321 The ``SourceDetectionTask`` "clipDetection" subtasks may support debug 1322 variables. See the documetation for `SourceDetectionTask` "clipDetection" 1323 for further information. 1327 `SafeClipAssembleCoaddTask` assembles a set of warped ``coaddTempExp`` 1328 images into a coadded image. The `SafeClipAssembleCoaddTask` is invoked by 1329 running assembleCoadd.py *without* the flag '--legacyCoadd'. 1331 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1332 and filter to be coadded (specified using 1333 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1334 along with a list of coaddTempExps to attempt to coadd (specified using 1335 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1336 Only the coaddTempExps that cover the specified tract and patch will be 1337 coadded. A list of the available optional arguments can be obtained by 1338 calling assembleCoadd.py with the --help command line argument: 1340 .. code-block:: none 1342 assembleCoadd.py --help 1344 To demonstrate usage of the `SafeClipAssembleCoaddTask` in the larger 1345 context of multi-band processing, we will generate the HSC-I & -R band 1346 coadds from HSC engineering test data provided in the ci_hsc package. 1347 To begin, assuming that the lsst stack has been already set up, we must 1348 set up the obs_subaru and ci_hsc packages. This defines the environment 1349 variable $CI_HSC_DIR and points at the location of the package. The raw 1350 HSC data live in the ``$CI_HSC_DIR/raw`` directory. To begin assembling 1351 the coadds, we must first 1354 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1356 create a skymap that covers the area of the sky present in the raw exposures 1357 - ``makeCoaddTempExp`` 1358 warp the individual calibrated exposures to the tangent plane of the coadd</DD> 1360 We can perform all of these steps by running 1362 .. code-block:: none 1364 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1366 This will produce warped coaddTempExps for each visit. To coadd the 1367 warped data, we call ``assembleCoadd.py`` as follows: 1369 .. code-block:: none 1371 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1372 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1373 --selectId visit=903986 ccd=100--selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1374 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1375 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1376 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1377 --selectId visit=903988 ccd=24 1379 This will process the HSC-I band data. The results are written in 1380 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1382 You may also choose to run: 1384 .. code-block:: none 1386 scons warp-903334 warp-903336 warp-903338 warp-903342 warp-903344 warp-903346 nnn 1387 assembleCoadd.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R --selectId visit=903334 ccd=16 \ 1388 --selectId visit=903334 ccd=22 --selectId visit=903334 ccd=23 --selectId visit=903334 ccd=100 \ 1389 --selectId visit=903336 ccd=17 --selectId visit=903336 ccd=24 --selectId visit=903338 ccd=18 \ 1390 --selectId visit=903338 ccd=25 --selectId visit=903342 ccd=4 --selectId visit=903342 ccd=10 \ 1391 --selectId visit=903342 ccd=100 --selectId visit=903344 ccd=0 --selectId visit=903344 ccd=5 \ 1392 --selectId visit=903344 ccd=11 --selectId visit=903346 ccd=1 --selectId visit=903346 ccd=6 \ 1393 --selectId visit=903346 ccd=12 1395 to generate the coadd for the HSC-R band if you are interested in following 1396 multiBand Coadd processing as discussed in ``pipeTasks_multiBand``. 1398 ConfigClass = SafeClipAssembleCoaddConfig
1399 _DefaultName =
"safeClipAssembleCoadd" 1402 AssembleCoaddTask.__init__(self, *args, **kwargs)
1403 schema = afwTable.SourceTable.makeMinimalSchema()
1404 self.makeSubtask(
"clipDetection", schema=schema)
1406 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, *args, **kwargs):
1407 """Assemble the coadd for a region. 1409 Compute the difference of coadds created with and without outlier 1410 rejection to identify coadd pixels that have outlier values in some 1412 Detect clipped regions on the difference image and mark these regions 1413 on the one or two individual coaddTempExps where they occur if there 1414 is significant overlap between the clipped region and a source. This 1415 leaves us with a set of footprints from the difference image that have 1416 been identified as having occured on just one or two individual visits. 1417 However, these footprints were generated from a difference image. It 1418 is conceivable for a large diffuse source to have become broken up 1419 into multiple footprints acrosss the coadd difference in this process. 1420 Determine the clipped region from all overlapping footprints from the 1421 detected sources in each visit - these are big footprints. 1422 Combine the small and big clipped footprints and mark them on a new 1424 Generate the coadd using `AssembleCoaddTask.run` without outlier 1425 removal. Clipped footprints will no longer make it into the coadd 1426 because they are marked in the new bad mask plane. 1430 skyInfo : `lsst.pipe.base.Struct` 1431 Patch geometry information, from getSkyInfo 1432 tempExpRefList : `list` 1433 List of data reference to tempExp 1434 imageScalerList : `list` 1435 List of image scalers 1441 result : `lsst.pipe.base.Struct` 1442 Result struct with components: 1444 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 1445 - ``nImage``: exposure count image (``lsst.afw.image.Image``). 1449 args and kwargs are passed but ignored in order to match the call 1450 signature expected by the parent task. 1453 mask = exp.getMaskedImage().getMask()
1454 mask.addMaskPlane(
"CLIPPED")
1456 result = self.
detectClip(exp, tempExpRefList)
1458 self.log.info(
'Found %d clipped objects', len(result.clipFootprints))
1460 maskClipValue = mask.getPlaneBitMask(
"CLIPPED")
1461 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1463 bigFootprints = self.
detectClipBig(result.clipSpans, result.clipFootprints, result.clipIndices,
1464 result.detectionFootprints, maskClipValue, maskDetValue,
1467 maskClip = mask.Factory(mask.getBBox(afwImage.PARENT))
1468 afwDet.setMaskFromFootprintList(maskClip, result.clipFootprints, maskClipValue)
1470 maskClipBig = maskClip.Factory(mask.getBBox(afwImage.PARENT))
1471 afwDet.setMaskFromFootprintList(maskClipBig, bigFootprints, maskClipValue)
1472 maskClip |= maskClipBig
1475 badMaskPlanes = self.config.badMaskPlanes[:]
1476 badMaskPlanes.append(
"CLIPPED")
1477 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
1478 return AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
1479 result.clipSpans, mask=badPixelMask)
1482 """Return an exposure that contains the difference between unclipped 1485 Generate a difference image between clipped and unclipped coadds. 1486 Compute the difference image by subtracting an outlier-clipped coadd 1487 from an outlier-unclipped coadd. Return the difference image. 1491 skyInfo : `lsst.pipe.base.Struct` 1492 Patch geometry information, from getSkyInfo 1493 tempExpRefList : `list` 1494 List of data reference to tempExp 1495 imageScalerList : `list` 1496 List of image scalers 1502 exp : `lsst.afw.image.Exposure` 1503 Difference image of unclipped and clipped coadd wrapped in an Exposure 1508 configIntersection = {k: getattr(self.config, k)
1509 for k, v
in self.config.toDict().items()
if (k
in config.keys())}
1510 config.update(**configIntersection)
1513 config.statistic =
'MEAN' 1515 coaddMean = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1517 config.statistic =
'MEANCLIP' 1519 coaddClip = task.run(skyInfo, tempExpRefList, imageScalerList, weightList).coaddExposure
1521 coaddDiff = coaddMean.getMaskedImage().Factory(coaddMean.getMaskedImage())
1522 coaddDiff -= coaddClip.getMaskedImage()
1523 exp = afwImage.ExposureF(coaddDiff)
1524 exp.setPsf(coaddMean.getPsf())
1528 """Detect clipped regions on an exposure and set the mask on the 1529 individual tempExp masks. 1531 Detect footprints in the difference image after smoothing the 1532 difference image with a Gaussian kernal. Identify footprints that 1533 overlap with one or two input ``coaddTempExps`` by comparing the 1534 computed overlap fraction to thresholds set in the config. A different 1535 threshold is applied depending on the number of overlapping visits 1536 (restricted to one or two). If the overlap exceeds the thresholds, 1537 the footprint is considered "CLIPPED" and is marked as such on the 1538 coaddTempExp. Return a struct with the clipped footprints, the indices 1539 of the ``coaddTempExps`` that end up overlapping with the clipped 1540 footprints, and a list of new masks for the ``coaddTempExps``. 1544 exp : `lsst.afw.image.Exposure` 1545 Exposure to run detection on. 1546 tempExpRefList : `list` 1547 List of data reference to tempExp. 1551 result : `lsst.pipe.base.Struct` 1552 Result struct with components: 1554 - ``clipFootprints``: list of clipped footprints. 1555 - ``clipIndices``: indices for each ``clippedFootprint`` in 1557 - ``clipSpans``: List of dictionaries containing spanSet lists 1558 to clip. Each element contains the new maskplane name 1559 ("CLIPPED") as the key and list of ``SpanSets`` as the value. 1560 - ``detectionFootprints``: List of DETECTED/DETECTED_NEGATIVE plane 1561 compressed into footprints. 1563 mask = exp.getMaskedImage().getMask()
1564 maskDetValue = mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE")
1565 fpSet = self.clipDetection.detectFootprints(exp, doSmooth=
True, clearMask=
True)
1567 fpSet.positive.merge(fpSet.negative)
1568 footprints = fpSet.positive
1569 self.log.info(
'Found %d potential clipped objects', len(footprints.getFootprints()))
1574 artifactSpanSets = [{
'CLIPPED': list()}
for _
in tempExpRefList]
1577 visitDetectionFootprints = []
1579 dims = [len(tempExpRefList), len(footprints.getFootprints())]
1580 overlapDetArr = numpy.zeros(dims, dtype=numpy.uint16)
1581 ignoreArr = numpy.zeros(dims, dtype=numpy.uint16)
1584 for i, warpRef
in enumerate(tempExpRefList):
1586 immediate=
True).getMaskedImage().getMask()
1587 maskVisitDet = tmpExpMask.Factory(tmpExpMask, tmpExpMask.getBBox(afwImage.PARENT),
1588 afwImage.PARENT,
True)
1589 maskVisitDet &= maskDetValue
1590 visitFootprints = afwDet.FootprintSet(maskVisitDet, afwDet.Threshold(1))
1591 visitDetectionFootprints.append(visitFootprints)
1593 for j, footprint
in enumerate(footprints.getFootprints()):
1598 for j, footprint
in enumerate(footprints.getFootprints()):
1599 nPixel = footprint.getArea()
1602 for i
in range(len(tempExpRefList)):
1603 ignore = ignoreArr[i, j]
1604 overlapDet = overlapDetArr[i, j]
1605 totPixel = nPixel - ignore
1608 if ignore > overlapDet
or totPixel <= 0.5*nPixel
or overlapDet == 0:
1610 overlap.append(overlapDet/float(totPixel))
1613 overlap = numpy.array(overlap)
1614 if not len(overlap):
1621 if len(overlap) == 1:
1622 if overlap[0] > self.config.minClipFootOverlapSingle:
1627 clipIndex = numpy.where(overlap > self.config.minClipFootOverlap)[0]
1628 if len(clipIndex) == 1:
1630 keepIndex = [clipIndex[0]]
1633 clipIndex = numpy.where(overlap > self.config.minClipFootOverlapDouble)[0]
1634 if len(clipIndex) == 2
and len(overlap) > 3:
1635 clipIndexComp = numpy.where(overlap <= self.config.minClipFootOverlapDouble)[0]
1636 if numpy.max(overlap[clipIndexComp]) <= self.config.maxClipFootOverlapDouble:
1638 keepIndex = clipIndex
1643 for index
in keepIndex:
1644 globalIndex = indexList[index]
1645 artifactSpanSets[globalIndex][
'CLIPPED'].append(footprint.spans)
1647 clipIndices.append(numpy.array(indexList)[keepIndex])
1648 clipFootprints.append(footprint)
1650 return pipeBase.Struct(clipFootprints=clipFootprints, clipIndices=clipIndices,
1651 clipSpans=artifactSpanSets, detectionFootprints=visitDetectionFootprints)
1653 def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints,
1654 maskClipValue, maskDetValue, coaddBBox):
1655 """Return individual warp footprints for large artifacts and append 1656 them to ``clipList`` in place. 1658 Identify big footprints composed of many sources in the coadd 1659 difference that may have originated in a large diffuse source in the 1660 coadd. We do this by indentifying all clipped footprints that overlap 1661 significantly with each source in all the coaddTempExps. 1666 List of alt mask SpanSets with clipping information. Modified. 1667 clipFootprints : `list` 1668 List of clipped footprints. 1669 clipIndices : `list` 1670 List of which entries in tempExpClipList each footprint belongs to. 1672 Mask value of clipped pixels. 1674 Mask value of detected pixels. 1675 coaddBBox : `lsst.afw.geom.Box` 1676 BBox of the coadd and warps. 1680 bigFootprintsCoadd : `list` 1681 List of big footprints 1683 bigFootprintsCoadd = []
1685 for index, (clippedSpans, visitFootprints)
in enumerate(zip(clipList, detectionFootprints)):
1686 maskVisitDet = afwImage.MaskX(coaddBBox, 0x0)
1687 for footprint
in visitFootprints.getFootprints():
1688 footprint.spans.setMask(maskVisitDet, maskDetValue)
1691 clippedFootprintsVisit = []
1692 for foot, clipIndex
in zip(clipFootprints, clipIndices):
1693 if index
not in clipIndex:
1695 clippedFootprintsVisit.append(foot)
1696 maskVisitClip = maskVisitDet.Factory(maskVisitDet.getBBox(afwImage.PARENT))
1697 afwDet.setMaskFromFootprintList(maskVisitClip, clippedFootprintsVisit, maskClipValue)
1699 bigFootprintsVisit = []
1700 for foot
in visitFootprints.getFootprints():
1701 if foot.getArea() < self.config.minBigOverlap:
1704 if nCount > self.config.minBigOverlap:
1705 bigFootprintsVisit.append(foot)
1706 bigFootprintsCoadd.append(foot)
1708 for footprint
in bigFootprintsVisit:
1709 clippedSpans[
"CLIPPED"].append(footprint.spans)
1711 return bigFootprintsCoadd
1715 assembleStaticSkyModel = pexConfig.ConfigurableField(
1716 target=AssembleCoaddTask,
1717 doc=
"Task to assemble an artifact-free, PSF-matched Coadd to serve as a" 1718 " naive/first-iteration model of the static sky.",
1720 detect = pexConfig.ConfigurableField(
1721 target=SourceDetectionTask,
1722 doc=
"Detect outlier sources on difference between each psfMatched warp and static sky model" 1724 detectTemplate = pexConfig.ConfigurableField(
1725 target=SourceDetectionTask,
1726 doc=
"Detect sources on static sky model. Only used if doPreserveContainedBySource is True" 1728 maxNumEpochs = pexConfig.Field(
1729 doc=
"Charactistic maximum local number of epochs/visits in which an artifact candidate can appear " 1730 "and still be masked. The effective maxNumEpochs is a broken linear function of local " 1731 "number of epochs (N): min(maxFractionEpochsLow*N, maxNumEpochs + maxFractionEpochsHigh*N). " 1732 "For each footprint detected on the image difference between the psfMatched warp and static sky " 1733 "model, if a significant fraction of pixels (defined by spatialThreshold) are residuals in more " 1734 "than the computed effective maxNumEpochs, the artifact candidate is deemed persistant rather " 1735 "than transient and not masked.",
1739 maxFractionEpochsLow = pexConfig.RangeField(
1740 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for low N. " 1741 "Effective maxNumEpochs = " 1742 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1747 maxFractionEpochsHigh = pexConfig.RangeField(
1748 doc=
"Fraction of local number of epochs (N) to use as effective maxNumEpochs for high N. " 1749 "Effective maxNumEpochs = " 1750 "min(maxFractionEpochsLow * N, maxNumEpochs + maxFractionEpochsHigh * N)",
1755 spatialThreshold = pexConfig.RangeField(
1756 doc=
"Unitless fraction of pixels defining how much of the outlier region has to meet the " 1757 "temporal criteria. If 0, clip all. If 1, clip none.",
1761 inclusiveMin=
True, inclusiveMax=
True 1763 doScaleWarpVariance = pexConfig.Field(
1764 doc=
"Rescale Warp variance plane using empirical noise?",
1768 scaleWarpVariance = pexConfig.ConfigurableField(
1769 target=ScaleVarianceTask,
1770 doc=
"Rescale variance on warps",
1772 doPreserveContainedBySource = pexConfig.Field(
1773 doc=
"Rescue artifacts from clipping that completely lie within a footprint detected" 1774 "on the PsfMatched Template Coadd. Replicates a behavior of SafeClip.",
1778 doPrefilterArtifacts = pexConfig.Field(
1779 doc=
"Ignore artifact candidates that are mostly covered by the bad pixel mask, " 1780 "because they will be excluded anyway. This prevents them from contributing " 1781 "to the outlier epoch count image and potentially being labeled as persistant." 1782 "'Mostly' is defined by the config 'prefilterArtifactsRatio'.",
1786 prefilterArtifactsMaskPlanes = pexConfig.ListField(
1787 doc=
"Prefilter artifact candidates that are mostly covered by these bad mask planes.",
1789 default=(
'NO_DATA',
'BAD',
'SAT',
'SUSPECT'),
1791 prefilterArtifactsRatio = pexConfig.Field(
1792 doc=
"Prefilter artifact candidates with less than this fraction overlapping good pixels",
1796 psfMatchedWarps = pipeBase.InputDatasetField(
1797 doc=(
"PSF-Matched Warps are required by CompareWarp regardless of the coadd type requested. " 1798 "Only PSF-Matched Warps make sense for image subtraction. " 1799 "Therefore, they must be in the InputDatasetField and made available to the task."),
1800 nameTemplate=
"{inputCoaddName}Coadd_psfMatchedWarp",
1801 storageClass=
"ExposureF",
1802 dimensions=(
"Tract",
"Patch",
"SkyMap",
"Visit"),
1807 AssembleCoaddConfig.setDefaults(self)
1823 self.
detect.doTempLocalBackground =
False 1824 self.
detect.reEstimateBackground =
False 1825 self.
detect.returnOriginalFootprints =
False 1826 self.
detect.thresholdPolarity =
"both" 1827 self.
detect.thresholdValue = 5
1828 self.
detect.nSigmaToGrow = 2
1829 self.
detect.minPixels = 4
1830 self.
detect.isotropicGrow =
True 1831 self.
detect.thresholdType =
"pixel_stdev" 1839 """Assemble a compareWarp coadded image from a set of warps 1840 by masking artifacts detected by comparing PSF-matched warps. 1842 In ``AssembleCoaddTask``, we compute the coadd as an clipped mean (i.e., 1843 we clip outliers). The problem with doing this is that when computing the 1844 coadd PSF at a given location, individual visit PSFs from visits with 1845 outlier pixels contribute to the coadd PSF and cannot be treated correctly. 1846 In this task, we correct for this behavior by creating a new badMaskPlane 1847 'CLIPPED' which marks pixels in the individual warps suspected to contain 1848 an artifact. We populate this plane on the input warps by comparing 1849 PSF-matched warps with a PSF-matched median coadd which serves as a 1850 model of the static sky. Any group of pixels that deviates from the 1851 PSF-matched template coadd by more than config.detect.threshold sigma, 1852 is an artifact candidate. The candidates are then filtered to remove 1853 variable sources and sources that are difficult to subtract such as 1854 bright stars. This filter is configured using the config parameters 1855 ``temporalThreshold`` and ``spatialThreshold``. The temporalThreshold is 1856 the maximum fraction of epochs that the deviation can appear in and still 1857 be considered an artifact. The spatialThreshold is the maximum fraction of 1858 pixels in the footprint of the deviation that appear in other epochs 1859 (where other epochs is defined by the temporalThreshold). If the deviant 1860 region meets this criteria of having a significant percentage of pixels 1861 that deviate in only a few epochs, these pixels have the 'CLIPPED' bit 1862 set in the mask. These regions will not contribute to the final coadd. 1863 Furthermore, any routine to determine the coadd PSF can now be cognizant 1864 of clipped regions. Note that the algorithm implemented by this task is 1865 preliminary and works correctly for HSC data. Parameter modifications and 1866 or considerable redesigning of the algorithm is likley required for other 1869 ``CompareWarpAssembleCoaddTask`` sub-classes 1870 ``AssembleCoaddTask`` and instantiates ``AssembleCoaddTask`` 1871 as a subtask to generate the TemplateCoadd (the model of the static sky). 1875 The `lsst.pipe.base.cmdLineTask.CmdLineTask` interface supports a 1876 flag ``-d`` to import ``debug.py`` from your ``PYTHONPATH``; see 1877 ``baseDebug`` for more about ``debug.py`` files. 1879 This task supports the following debug variables: 1882 If True then save the Epoch Count Image as a fits file in the `figPath` 1884 Path to save the debug fits images and figures 1886 For example, put something like: 1888 .. code-block:: python 1891 def DebugInfo(name): 1892 di = lsstDebug.getInfo(name) 1893 if name == "lsst.pipe.tasks.assembleCoadd": 1894 di.saveCountIm = True 1895 di.figPath = "/desired/path/to/debugging/output/images" 1897 lsstDebug.Info = DebugInfo 1899 into your ``debug.py`` file and run ``assemebleCoadd.py`` with the 1900 ``--debug`` flag. Some subtasks may have their own debug variables; 1901 see individual Task documentation. 1905 ``CompareWarpAssembleCoaddTask`` assembles a set of warped images into a 1906 coadded image. The ``CompareWarpAssembleCoaddTask`` is invoked by running 1907 ``assembleCoadd.py`` with the flag ``--compareWarpCoadd``. 1908 Usage of ``assembleCoadd.py`` expects a data reference to the tract patch 1909 and filter to be coadded (specified using 1910 '--id = [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]') 1911 along with a list of coaddTempExps to attempt to coadd (specified using 1912 '--selectId [KEY=VALUE1[^VALUE2[^VALUE3...] [KEY=VALUE1[^VALUE2[^VALUE3...] ...]]'). 1913 Only the warps that cover the specified tract and patch will be coadded. 1914 A list of the available optional arguments can be obtained by calling 1915 ``assembleCoadd.py`` with the ``--help`` command line argument: 1917 .. code-block:: none 1919 assembleCoadd.py --help 1921 To demonstrate usage of the ``CompareWarpAssembleCoaddTask`` in the larger 1922 context of multi-band processing, we will generate the HSC-I & -R band 1923 oadds from HSC engineering test data provided in the ``ci_hsc`` package. 1924 To begin, assuming that the lsst stack has been already set up, we must 1925 set up the ``obs_subaru`` and ``ci_hsc`` packages. 1926 This defines the environment variable ``$CI_HSC_DIR`` and points at the 1927 location of the package. The raw HSC data live in the ``$CI_HSC_DIR/raw`` 1928 directory. To begin assembling the coadds, we must first 1931 process the individual ccds in $CI_HSC_RAW to produce calibrated exposures 1933 create a skymap that covers the area of the sky present in the raw exposures 1935 warp the individual calibrated exposures to the tangent plane of the coadd 1937 We can perform all of these steps by running 1939 .. code-block:: none 1941 $CI_HSC_DIR scons warp-903986 warp-904014 warp-903990 warp-904010 warp-903988 1943 This will produce warped ``coaddTempExps`` for each visit. To coadd the 1944 warped data, we call ``assembleCoadd.py`` as follows: 1946 .. code-block:: none 1948 assembleCoadd.py --compareWarpCoadd $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I \ 1949 --selectId visit=903986 ccd=16 --selectId visit=903986 ccd=22 --selectId visit=903986 ccd=23 \ 1950 --selectId visit=903986 ccd=100 --selectId visit=904014 ccd=1 --selectId visit=904014 ccd=6 \ 1951 --selectId visit=904014 ccd=12 --selectId visit=903990 ccd=18 --selectId visit=903990 ccd=25 \ 1952 --selectId visit=904010 ccd=4 --selectId visit=904010 ccd=10 --selectId visit=904010 ccd=100 \ 1953 --selectId visit=903988 ccd=16 --selectId visit=903988 ccd=17 --selectId visit=903988 ccd=23 \ 1954 --selectId visit=903988 ccd=24 1956 This will process the HSC-I band data. The results are written in 1957 ``$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I``. 1959 ConfigClass = CompareWarpAssembleCoaddConfig
1960 _DefaultName =
"compareWarpAssembleCoadd" 1963 AssembleCoaddTask.__init__(self, *args, **kwargs)
1964 self.makeSubtask(
"assembleStaticSkyModel")
1965 detectionSchema = afwTable.SourceTable.makeMinimalSchema()
1966 self.makeSubtask(
"detect", schema=detectionSchema)
1967 if self.config.doPreserveContainedBySource:
1968 self.makeSubtask(
"detectTemplate", schema=afwTable.SourceTable.makeMinimalSchema())
1969 if self.config.doScaleWarpVariance:
1970 self.makeSubtask(
"scaleWarpVariance")
1973 """Make inputs specific to Subclass with Gen 3 API 1975 Calls Gen3 `adaptArgsAndRun` instead of the Gen2 specific `runDataRef` 1977 Duplicates interface of`adaptArgsAndRun` method. 1978 Available to be implemented by subclasses only if they need the 1979 coadd dataRef for performing preliminary processing before 1980 assembling the coadd. 1985 Keys are the names of the configs describing input dataset types. 1986 Values are input Python-domain data objects (or lists of objects) 1987 retrieved from data butler. 1988 inputDataIds : `dict` 1989 Keys are the names of the configs describing input dataset types. 1990 Values are DataIds (or lists of DataIds) that task consumes for 1991 corresponding dataset type. 1992 DataIds are guaranteed to match data objects in ``inputData``. 1993 outputDataIds : `dict` 1994 Keys are the names of the configs describing input dataset types. 1995 Values are DataIds (or lists of DataIds) that task is to produce 1996 for corresponding dataset type. 1997 butler : `lsst.daf.butler.Butler` 1998 Gen3 Butler object for fetching additional data products before 2003 result : `lsst.pipe.base.Struct` 2004 Result struct with components: 2006 - ``templateCoadd`` : coadded exposure (``lsst.afw.image.Exposure``) 2007 - ``nImage``: N Image (``lsst.afw.image.Image``) 2011 templateCoadd = self.assembleStaticSkyModel.
adaptArgsAndRun(inputData, inputDataIds,
2012 outputDataIds, butler)
2013 if templateCoadd
is None:
2016 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2017 nImage=templateCoadd.nImage)
2020 """Make inputs specific to Subclass. 2022 Generate a templateCoadd to use as a native model of static sky to 2023 subtract from warps. 2027 dataRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2028 Butler dataRef for supplementary data. 2029 selectDataList : `list` (optional) 2030 Optional List of data references to Calexps. 2031 warpRefList : `list` (optional) 2032 Optional List of data references to Warps. 2036 result : `lsst.pipe.base.Struct` 2037 Result struct with components: 2039 - ``templateCoadd``: coadded exposure (``lsst.afw.image.Exposure``) 2040 - ``nImage``: N Image (``lsst.afw.image.Image``) 2042 templateCoadd = self.assembleStaticSkyModel.
runDataRef(dataRef, selectDataList, warpRefList)
2043 if templateCoadd
is None:
2046 return pipeBase.Struct(templateCoadd=templateCoadd.coaddExposure,
2047 nImage=templateCoadd.nImage)
2049 def _noTemplateMessage(self, warpType):
2050 warpName = (warpType[0].upper() + warpType[1:])
2051 message =
"""No %(warpName)s warps were found to build the template coadd which is 2052 required to run CompareWarpAssembleCoaddTask. To continue assembling this type of coadd, 2053 first either rerun makeCoaddTempExp with config.make%(warpName)s=True or 2054 coaddDriver with config.makeCoadTempExp.make%(warpName)s=True, before assembleCoadd. 2056 Alternatively, to use another algorithm with existing warps, retarget the CoaddDriverConfig to 2057 another algorithm like: 2059 from lsst.pipe.tasks.assembleCoadd import SafeClipAssembleCoaddTask 2060 config.assemble.retarget(SafeClipAssembleCoaddTask) 2061 """ % {
"warpName": warpName}
2064 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2065 supplementaryData, *args, **kwargs):
2066 """Assemble the coadd. 2068 Find artifacts and apply them to the warps' masks creating a list of 2069 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" 2070 plane. Then pass these alternative masks to the base class's `run` 2075 skyInfo : `lsst.pipe.base.Struct` 2076 Patch geometry information. 2077 tempExpRefList : `list` 2078 List of data references to warps. 2079 imageScalerList : `list` 2080 List of image scalers. 2083 supplementaryData : `lsst.pipe.base.Struct` 2084 This Struct must contain a ``templateCoadd`` that serves as the 2085 model of the static sky. 2089 result : `lsst.pipe.base.Struct` 2090 Result struct with components: 2092 - ``coaddExposure``: coadded exposure (``lsst.afw.image.Exposure``). 2093 - ``nImage``: exposure count image (``lsst.afw.image.Image``), if requested. 2095 templateCoadd = supplementaryData.templateCoadd
2096 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
2097 badMaskPlanes = self.config.badMaskPlanes[:]
2098 badMaskPlanes.append(
"CLIPPED")
2099 badPixelMask = afwImage.Mask.getPlaneBitMask(badMaskPlanes)
2101 result = AssembleCoaddTask.run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
2102 spanSetMaskList, mask=badPixelMask)
2106 self.
applyAltEdgeMask(result.coaddExposure.maskedImage.mask, spanSetMaskList)
2110 """Propagate alt EDGE mask to SENSOR_EDGE AND INEXACT_PSF planes. 2114 mask : `lsst.afw.image.Mask` 2116 altMaskList : `list` 2117 List of Dicts containing ``spanSet`` lists. 2118 Each element contains the new mask plane name (e.g. "CLIPPED 2119 and/or "NO_DATA") as the key, and list of ``SpanSets`` to apply to 2122 maskValue = mask.getPlaneBitMask([
"SENSOR_EDGE",
"INEXACT_PSF"])
2123 for visitMask
in altMaskList:
2124 if "EDGE" in visitMask:
2125 for spanSet
in visitMask[
'EDGE']:
2126 spanSet.clippedTo(mask.getBBox()).setMask(mask, maskValue)
2131 Loop through warps twice. The first loop builds a map with the count 2132 of how many epochs each pixel deviates from the templateCoadd by more 2133 than ``config.chiThreshold`` sigma. The second loop takes each 2134 difference image and filters the artifacts detected in each using 2135 count map to filter out variable sources and sources that are 2136 difficult to subtract cleanly. 2140 templateCoadd : `lsst.afw.image.Exposure` 2141 Exposure to serve as model of static sky. 2142 tempExpRefList : `list` 2143 List of data references to warps. 2144 imageScalerList : `list` 2145 List of image scalers. 2150 List of dicts containing information about CLIPPED 2151 (i.e., artifacts), NO_DATA, and EDGE pixels. 2154 self.log.debug(
"Generating Count Image, and mask lists.")
2155 coaddBBox = templateCoadd.getBBox()
2156 slateIm = afwImage.ImageU(coaddBBox)
2157 epochCountImage = afwImage.ImageU(coaddBBox)
2158 nImage = afwImage.ImageU(coaddBBox)
2159 spanSetArtifactList = []
2160 spanSetNoDataMaskList = []
2161 spanSetEdgeList = []
2165 templateCoadd.mask.clearAllMaskPlanes()
2167 if self.config.doPreserveContainedBySource:
2168 templateFootprints = self.detectTemplate.detectFootprints(templateCoadd)
2170 templateFootprints =
None 2172 for warpRef, imageScaler
in zip(tempExpRefList, imageScalerList):
2174 if warpDiffExp
is not None:
2176 nImage.array += (numpy.isfinite(warpDiffExp.image.array) *
2177 ((warpDiffExp.mask.array & badPixelMask) == 0)).astype(numpy.uint16)
2178 fpSet = self.detect.detectFootprints(warpDiffExp, doSmooth=
False, clearMask=
True)
2179 fpSet.positive.merge(fpSet.negative)
2180 footprints = fpSet.positive
2182 spanSetList = [footprint.spans
for footprint
in footprints.getFootprints()]
2185 if self.config.doPrefilterArtifacts:
2187 for spans
in spanSetList:
2188 spans.setImage(slateIm, 1, doClip=
True)
2189 epochCountImage += slateIm
2195 nans = numpy.where(numpy.isnan(warpDiffExp.maskedImage.image.array), 1, 0)
2196 nansMask = afwImage.makeMaskFromArray(nans.astype(afwImage.MaskPixel))
2197 nansMask.setXY0(warpDiffExp.getXY0())
2198 edgeMask = warpDiffExp.mask
2199 spanSetEdgeMask = afwGeom.SpanSet.fromMask(edgeMask,
2200 edgeMask.getPlaneBitMask(
"EDGE")).split()
2204 nansMask = afwImage.MaskX(coaddBBox, 1)
2206 spanSetEdgeMask = []
2208 spanSetNoDataMask = afwGeom.SpanSet.fromMask(nansMask).split()
2210 spanSetNoDataMaskList.append(spanSetNoDataMask)
2211 spanSetArtifactList.append(spanSetList)
2212 spanSetEdgeList.append(spanSetEdgeMask)
2216 epochCountImage.writeFits(path)
2218 for i, spanSetList
in enumerate(spanSetArtifactList):
2220 filteredSpanSetList = self.
filterArtifacts(spanSetList, epochCountImage, nImage,
2222 spanSetArtifactList[i] = filteredSpanSetList
2225 for artifacts, noData, edge
in zip(spanSetArtifactList, spanSetNoDataMaskList, spanSetEdgeList):
2226 altMasks.append({
'CLIPPED': artifacts,
2232 """Remove artifact candidates covered by bad mask plane. 2234 Any future editing of the candidate list that does not depend on 2235 temporal information should go in this method. 2239 spanSetList : `list` 2240 List of SpanSets representing artifact candidates. 2241 exp : `lsst.afw.image.Exposure` 2242 Exposure containing mask planes used to prefilter. 2246 returnSpanSetList : `list` 2247 List of SpanSets with artifacts. 2249 badPixelMask = exp.mask.getPlaneBitMask(self.config.prefilterArtifactsMaskPlanes)
2250 goodArr = (exp.mask.array & badPixelMask) == 0
2251 returnSpanSetList = []
2252 bbox = exp.getBBox()
2253 x0, y0 = exp.getXY0()
2254 for i, span
in enumerate(spanSetList):
2255 y, x = span.clippedTo(bbox).indices()
2256 yIndexLocal = numpy.array(y) - y0
2257 xIndexLocal = numpy.array(x) - x0
2258 goodRatio = numpy.count_nonzero(goodArr[yIndexLocal, xIndexLocal])/span.getArea()
2259 if goodRatio > self.config.prefilterArtifactsRatio:
2260 returnSpanSetList.append(span)
2261 return returnSpanSetList
2263 def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None):
2264 """Filter artifact candidates. 2268 spanSetList : `list` 2269 List of SpanSets representing artifact candidates. 2270 epochCountImage : `lsst.afw.image.Image` 2271 Image of accumulated number of warpDiff detections. 2272 nImage : `lsst.afw.image.Image` 2273 Image of the accumulated number of total epochs contributing. 2277 maskSpanSetList : `list` 2278 List of SpanSets with artifacts. 2281 maskSpanSetList = []
2282 x0, y0 = epochCountImage.getXY0()
2283 for i, span
in enumerate(spanSetList):
2284 y, x = span.indices()
2285 yIdxLocal = [y1 - y0
for y1
in y]
2286 xIdxLocal = [x1 - x0
for x1
in x]
2287 outlierN = epochCountImage.array[yIdxLocal, xIdxLocal]
2288 totalN = nImage.array[yIdxLocal, xIdxLocal]
2291 effMaxNumEpochsHighN = (self.config.maxNumEpochs +
2292 self.config.maxFractionEpochsHigh*numpy.mean(totalN))
2293 effMaxNumEpochsLowN = self.config.maxFractionEpochsLow * numpy.mean(totalN)
2294 effectiveMaxNumEpochs = int(min(effMaxNumEpochsLowN, effMaxNumEpochsHighN))
2295 nPixelsBelowThreshold = numpy.count_nonzero((outlierN > 0) &
2296 (outlierN <= effectiveMaxNumEpochs))
2297 percentBelowThreshold = nPixelsBelowThreshold / len(outlierN)
2298 if percentBelowThreshold > self.config.spatialThreshold:
2299 maskSpanSetList.append(span)
2301 if self.config.doPreserveContainedBySource
and footprintsToExclude
is not None:
2303 filteredMaskSpanSetList = []
2304 for span
in maskSpanSetList:
2306 for footprint
in footprintsToExclude.positive.getFootprints():
2307 if footprint.spans.contains(span):
2311 filteredMaskSpanSetList.append(span)
2312 maskSpanSetList = filteredMaskSpanSetList
2314 return maskSpanSetList
2316 def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd):
2317 """Fetch a warp from the butler and return a warpDiff. 2321 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2322 Butler dataRef for the warp. 2323 imageScaler : `lsst.pipe.tasks.scaleZeroPoint.ImageScaler` 2324 An image scaler object. 2325 templateCoadd : `lsst.afw.image.Exposure` 2326 Exposure to be substracted from the scaled warp. 2330 warp : `lsst.afw.image.Exposure` 2331 Exposure of the image difference between the warp and template. 2336 if not warpRef.datasetExists(warpName):
2337 self.log.warn(
"Could not find %s %s; skipping it", warpName, warpRef.dataId)
2339 warp = warpRef.get(warpName, immediate=
True)
2341 imageScaler.scaleMaskedImage(warp.getMaskedImage())
2342 mi = warp.getMaskedImage()
2343 if self.config.doScaleWarpVariance:
2345 self.scaleWarpVariance.
run(mi)
2346 except Exception
as exc:
2347 self.log.warn(
"Unable to rescale variance of warp (%s); leaving it as-is" % (exc,))
2348 mi -= templateCoadd.getMaskedImage()
2351 def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False):
2352 """Return a path to which to write debugging output. 2354 Creates a hyphen-delimited string of dataId values for simple filenames. 2359 Prefix for filename. 2360 warpRef : `lsst.daf.persistence.butlerSubset.ButlerDataRef` 2361 Butler dataRef to make the path from. 2362 coaddLevel : `bool`, optional. 2363 If True, include only coadd-level keys (e.g., 'tract', 'patch', 2364 'filter', but no 'visit'). 2369 Path for debugging output. 2374 keys = warpRef.dataId.keys()
2375 keyList = sorted(keys, reverse=
True)
2377 filename =
"%s-%s.fits" % (prefix,
'-'.join([str(warpRef.dataId[k])
for k
in keyList]))
2378 return os.path.join(directory, filename)
def setBrightObjectMasks(self, exposure, dataId, brightObjectMasks)
def shrinkValidPolygons(self, coaddInputs)
def getCoaddDatasetName(self, warpType="direct")
def _dataRef2DebugPath(self, prefix, warpRef, coaddLevel=False)
def getGroupDataRef(butler, datasetType, groupTuple, keys)
Base class for coaddition.
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def getTempExpRefList(self, patchRef, calExpRefList)
def removeMaskPlanes(self, maskedImage)
def makeSkyInfo(skyMap, tractId, patchId)
def _readAndComputeWarpDiff(self, warpRef, imageScaler, templateCoadd)
def prepareInputs(self, refList)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def __init__(self, args, kwargs)
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def prepareStats(self, mask=None)
def makeDataRefList(self, namespace)
def getBadPixelMask(self)
Convenience method to provide the bitmask from the mask plane names.
def assembleSubregion(self, coaddExposure, bbox, tempExpRefList, imageScalerList, weightList, altMaskList, statsFlags, statsCtrl, nImage=None)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def detectClip(self, exp, tempExpRefList)
def setInexactPsf(self, mask)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def __init__(self, args, kwargs)
def makeSupplementaryData(self, dataRef, selectDataList=None, warpRefList=None)
def getPrerequisiteDatasetTypes(cls, config)
def filterArtifacts(self, spanSetList, epochCountImage, nImage, footprintsToExclude=None)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, args, kwargs)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData, args, kwargs)
def buildDifferenceImage(self, skyInfo, tempExpRefList, imageScalerList, weightList)
def _noTemplateMessage(self, warpType)
def selectExposures(self, patchRef, skyInfo=None, selectDataList=[])
Select exposures to coadd.
def setRejectedMaskMapping(statsCtrl)
def getOutputDatasetTypes(cls, config)
def applyAltEdgeMask(self, mask, altMaskList)
def getInputDatasetTypes(cls, config)
def makeSupplementaryDataGen3(self, inputData, inputDataIds, outputDataIds, butler)
def readBrightObjectMasks(self, dataRef)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def processResults(self, coaddExposure, dataRef)
def __init__(self, args, kwargs)
def prefilterArtifacts(self, spanSetList, exp)
def _subBBoxIter(bbox, subregionSize)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def countMaskFromFootprint(mask, footprint, bitmask, ignoreMask)
def groupPatchExposures(patchDataRef, calexpDataRefList, coaddDatasetType="deepCoadd", tempExpDatasetType="deepCoadd_directWarp")
def detectClipBig(self, clipList, clipFootprints, clipIndices, detectionFootprints, maskClipValue, maskDetValue, coaddBBox)