Coverage for python/lsst/fgcmcal/fgcmBuildStarsBase.py: 18%

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

# This file is part of fgcmcal.

# Developed for the LSST Data Management System.

# This product includes software developed by the LSST Project

# (https://www.lsst.org).

# See the COPYRIGHT file at the top-level directory of this distribution

# for details of code ownership.

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License

# along with this program. If not, see <https://www.gnu.org/licenses/>.

"""Base class for BuildStars using src tables or sourceTable_visit tables.

"""

import os

import sys

import traceback

import abc

import numpy as np

import lsst.pex.config as pexConfig

import lsst.pipe.base as pipeBase

import lsst.afw.table as afwTable

import lsst.geom as geom

from lsst.daf.base import PropertyList

from lsst.daf.base.dateTime import DateTime

from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry

from .utilities import computeApertureRadiusFromDataRef

from .fgcmLoadReferenceCatalog import FgcmLoadReferenceCatalogTask

import fgcm

REFSTARS_FORMAT_VERSION = 1

__all__ = ['FgcmBuildStarsConfigBase', 'FgcmBuildStarsRunner', 'FgcmBuildStarsBaseTask']

class FgcmBuildStarsConfigBase(pexConfig.Config):

"""Base config for FgcmBuildStars tasks"""

instFluxField = pexConfig.Field(

doc=("Faull name of the source instFlux field to use, including 'instFlux'. "

"The associated flag will be implicitly included in badFlags"),

dtype=str,

default='slot_CalibFlux_instFlux',

)

minPerBand = pexConfig.Field(

doc="Minimum observations per band",

dtype=int,

default=2,

)

matchRadius = pexConfig.Field(

doc="Match radius (arcseconds)",

dtype=float,

default=1.0,

)

isolationRadius = pexConfig.Field(

doc="Isolation radius (arcseconds)",

dtype=float,

default=2.0,

)

densityCutNside = pexConfig.Field(

doc="Density cut healpix nside",

dtype=int,

default=128,

)

densityCutMaxPerPixel = pexConfig.Field(

doc="Density cut number of stars per pixel",

dtype=int,

default=1000,

)

matchNside = pexConfig.Field(

doc="Healpix Nside for matching",

dtype=int,

default=4096,

)

coarseNside = pexConfig.Field(

doc="Healpix coarse Nside for partitioning matches",

dtype=int,

default=8,

)

filterMap = pexConfig.DictField(

doc="Mapping from 'filterName' to band.",

keytype=str,

itemtype=str,

default={},

)

requiredBands = pexConfig.ListField(

doc="Bands required for each star",

dtype=str,

default=(),

)

primaryBands = pexConfig.ListField(

doc=("Bands for 'primary' star matches. "

"A star must be observed in one of these bands to be considered "

"as a calibration star."),

dtype=str,

default=None

)

visitDataRefName = pexConfig.Field(

doc="dataRef name for the 'visit' field, usually 'visit'.",

dtype=str,

default="visit"

)

ccdDataRefName = pexConfig.Field(

doc="dataRef name for the 'ccd' field, usually 'ccd' or 'detector'.",

dtype=str,

default="ccd"

)

doApplyWcsJacobian = pexConfig.Field(

doc="Apply the jacobian of the WCS to the star observations prior to fit?",

dtype=bool,

default=True

)

psfCandidateName = pexConfig.Field(

doc="Name of field with psf candidate flag for propagation",

dtype=str,

default="calib_psf_candidate"

)

doSubtractLocalBackground = pexConfig.Field(

doc=("Subtract the local background before performing calibration? "

"This is only supported for circular aperture calibration fluxes."),

dtype=bool,

default=False

)

localBackgroundFluxField = pexConfig.Field(

doc="Full name of the local background instFlux field to use.",

dtype=str,

default='base_LocalBackground_instFlux'

)

sourceSelector = sourceSelectorRegistry.makeField(

doc="How to select sources",

default="science"

)

apertureInnerInstFluxField = pexConfig.Field(

doc=("Full name of instFlux field that contains inner aperture "

"flux for aperture correction proxy"),

dtype=str,

default='base_CircularApertureFlux_12_0_instFlux'

)

apertureOuterInstFluxField = pexConfig.Field(

doc=("Full name of instFlux field that contains outer aperture "

"flux for aperture correction proxy"),

dtype=str,

default='base_CircularApertureFlux_17_0_instFlux'

)

doReferenceMatches = pexConfig.Field(

doc="Match reference catalog as additional constraint on calibration",

dtype=bool,

default=True,

)

fgcmLoadReferenceCatalog = pexConfig.ConfigurableField(

target=FgcmLoadReferenceCatalogTask,

doc="FGCM reference object loader",

)

nVisitsPerCheckpoint = pexConfig.Field(

doc="Number of visits read between checkpoints",

dtype=int,

default=500,

)

def setDefaults(self):

sourceSelector = self.sourceSelector["science"]

sourceSelector.setDefaults()

sourceSelector.doFlags = True

sourceSelector.doUnresolved = True

sourceSelector.doSignalToNoise = True

sourceSelector.doIsolated = True

sourceSelector.signalToNoise.minimum = 10.0

sourceSelector.signalToNoise.maximum = 1000.0

# FGCM operates on unresolved sources, and this setting is

# appropriate for the current base_ClassificationExtendedness

sourceSelector.unresolved.maximum = 0.5

class FgcmBuildStarsRunner(pipeBase.ButlerInitializedTaskRunner):

"""Subclass of TaskRunner for FgcmBuildStars tasks

fgcmBuildStarsTask.run() and fgcmBuildStarsTableTask.run() take a number of

arguments, one of which is the butler (for persistence and mapper data),

and a list of dataRefs extracted from the command line. Note that FGCM

runs on a large set of dataRefs, and not on single dataRef/tract/patch.

This class transforms the process arguments generated by the ArgumentParser

into the arguments expected by FgcmBuildStarsTask.run(). This runner does

not use any parallelization.

"""

@staticmethod

def getTargetList(parsedCmd):

"""

Return a list with one element: a tuple with the butler and

list of dataRefs

"""

# we want to combine the butler with any (or no!) dataRefs

return [(parsedCmd.butler, parsedCmd.id.refList)]

def __call__(self, args):

"""

Parameters

----------

args: `tuple` with (butler, dataRefList)

Returns

-------

exitStatus: `list` with `lsst.pipe.base.Struct`

exitStatus (0: success; 1: failure)

"""

butler, dataRefList = args

task = self.TaskClass(config=self.config, log=self.log)

exitStatus = 0

if self.doRaise:

task.runDataRef(butler, dataRefList)

else:

try:

task.runDataRef(butler, dataRefList)

except Exception as e:

exitStatus = 1

task.log.fatal("Failed: %s" % e)

if not isinstance(e, pipeBase.TaskError):

traceback.print_exc(file=sys.stderr)

task.writeMetadata(butler)

# The task does not return any results:

return [pipeBase.Struct(exitStatus=exitStatus)]

def run(self, parsedCmd):

"""

Run the task, with no multiprocessing

Parameters

----------

parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line

"""

resultList = []

if self.precall(parsedCmd):

targetList = self.getTargetList(parsedCmd)

resultList = self(targetList[0])

return resultList

class FgcmBuildStarsBaseTask(pipeBase.CmdLineTask, abc.ABC):

"""

Base task to build stars for FGCM global calibration

Parameters

----------

butler : `lsst.daf.persistence.Butler`

"""

def __init__(self, butler=None, **kwargs):

pipeBase.CmdLineTask.__init__(self, **kwargs)

self.makeSubtask("sourceSelector")

# Only log warning and fatal errors from the sourceSelector

self.sourceSelector.log.setLevel(self.sourceSelector.log.WARN)

# no saving of metadata for now

def _getMetadataName(self):

return None

@pipeBase.timeMethod

def runDataRef(self, butler, dataRefs):

"""

Cross-match and make star list for FGCM Input

Parameters

----------

butler: `lsst.daf.persistence.Butler`

dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`

Source data references for the input visits.

Raises

------

RuntimeErrror: Raised if `config.doReferenceMatches` is set and

an fgcmLookUpTable is not available, or if computeFluxApertureRadius()

fails if the calibFlux is not a CircularAperture flux.

"""

datasetType = dataRefs[0].butlerSubset.datasetType

self.log.info("Running with %d %s dataRefs", len(dataRefs), datasetType)

if self.config.doReferenceMatches:

# Ensure that we have a LUT

if not butler.datasetExists('fgcmLookUpTable'):

raise RuntimeError("Must have fgcmLookUpTable if using config.doReferenceMatches")

# Compute aperture radius if necessary. This is useful to do now before

# any heavy lifting has happened (fail early).

calibFluxApertureRadius = None

if self.config.doSubtractLocalBackground:

try:

calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0],

self.config.instFluxField)

except RuntimeError as e:

raise RuntimeError("Could not determine aperture radius from %s. "

"Cannot use doSubtractLocalBackground." %

(self.config.instFluxField)) from e

groupedDataRefs = self.findAndGroupDataRefs(butler, dataRefs)

camera = butler.get('camera')

# Make the visit catalog if necessary

# First check if the visit catalog is in the _current_ path

# We cannot use Gen2 datasetExists() because that checks all parent

# directories as well, which would make recovering from faults

# and fgcmcal reruns impossible.

visitCatDataRef = butler.dataRef('fgcmVisitCatalog')

filename = visitCatDataRef.get('fgcmVisitCatalog_filename')[0]

if os.path.exists(filename):

# This file exists and we should continue processing

inVisitCat = visitCatDataRef.get()

if len(inVisitCat) != len(groupedDataRefs):

raise RuntimeError("Existing visitCatalog found, but has an inconsistent "

"number of visits. Cannot continue.")

else:

inVisitCat = None

visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs,

visitCatDataRef=visitCatDataRef,

inVisitCat=inVisitCat)

# Persist the visitCat as a checkpoint file.

visitCatDataRef.put(visitCat)

starObsDataRef = butler.dataRef('fgcmStarObservations')

filename = starObsDataRef.get('fgcmStarObservations_filename')[0]

if os.path.exists(filename):

inStarObsCat = starObsDataRef.get()

else:

inStarObsCat = None

rad = calibFluxApertureRadius

fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs,

visitCat,

calibFluxApertureRadius=rad,

starObsDataRef=starObsDataRef,

visitCatDataRef=visitCatDataRef,

inStarObsCat=inStarObsCat)

visitCatDataRef.put(visitCat)

starObsDataRef.put(fgcmStarObservationCat)

# Always do the matching.

fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(butler,

visitCat,

fgcmStarObservationCat)

# Persist catalogs via the butler

butler.put(fgcmStarIdCat, 'fgcmStarIds')

butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')

if fgcmRefCat is not None:

butler.put(fgcmRefCat, 'fgcmReferenceStars')

@abc.abstractmethod

def findAndGroupDataRefs(self, butler, dataRefs):

"""

Find and group dataRefs (by visit).

Parameters

----------

butler: `lsst.daf.persistence.Butler`

dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`

Data references for the input visits.

Returns

-------

groupedDataRefs: `dict` [`int`, `list`]

Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef`

"""

raise NotImplementedError("findAndGroupDataRefs not implemented.")

@abc.abstractmethod

def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,

calibFluxApertureRadius=None,

visitCatDataRef=None,

starObsDataRef=None,

inStarObsCat=None):

"""

Compile all good star observations from visits in visitCat. Checkpoint files

will be stored if both visitCatDataRef and starObsDataRef are not None.

Parameters

----------

groupedDataRefs: `dict` of `list`s

Lists of `lsst.daf.persistence.ButlerDataRef`, grouped by visit.

visitCat: `afw.table.BaseCatalog`

Catalog with visit data for FGCM

calibFluxApertureRadius: `float`, optional

Aperture radius for calibration flux.

visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional

Dataref to write visitCat for checkpoints

starObsDataRef: `lsst.daf.persistence.ButlerDataRef`, optional

Dataref to write the star observation catalog for checkpoints.

inStarObsCat: `afw.table.BaseCatalog`

Input observation catalog. If this is incomplete, observations

will be appended from when it was cut off.

Returns

-------

fgcmStarObservations: `afw.table.BaseCatalog`

Full catalog of good observations.

Raises

------

RuntimeError: Raised if doSubtractLocalBackground is True and

calibFluxApertureRadius is not set.

"""

raise NotImplementedError("fgcmMakeAllStarObservations not implemented.")

def fgcmMakeVisitCatalog(self, camera, groupedDataRefs,

visitCatDataRef=None, inVisitCat=None):

"""

Make a visit catalog with all the keys from each visit

Parameters

----------

camera: `lsst.afw.cameraGeom.Camera`

Camera from the butler

groupedDataRefs: `dict`

Dictionary with visit keys, and `list`s of

`lsst.daf.persistence.ButlerDataRef`

visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional

Dataref to write visitCat for checkpoints

inVisitCat: `afw.table.BaseCatalog`

Input (possibly incomplete) visit catalog

Returns

-------

visitCat: `afw.table.BaseCatalog`

"""

self.log.info("Assembling visitCatalog from %d %ss" %

(len(groupedDataRefs), self.config.visitDataRefName))

nCcd = len(camera)

if inVisitCat is None:

schema = self._makeFgcmVisitSchema(nCcd)

visitCat = afwTable.BaseCatalog(schema)

visitCat.reserve(len(groupedDataRefs))

for i, visit in enumerate(sorted(groupedDataRefs)):

rec = visitCat.addNew()

rec['visit'] = visit

rec['used'] = 0

rec['sources_read'] = 0

else:

visitCat = inVisitCat

# No matter what, fill the catalog. This will check if it was

# already read.

self._fillVisitCatalog(visitCat, groupedDataRefs,

visitCatDataRef=visitCatDataRef)

return visitCat

def _fillVisitCatalog(self, visitCat, groupedDataRefs,

visitCatDataRef=None):

"""

Fill the visit catalog with visit metadata

Parameters

----------

visitCat: `afw.table.BaseCatalog`

Catalog with schema from _makeFgcmVisitSchema()

groupedDataRefs: `dict`

Dictionary with visit keys, and `list`s of

`lsst.daf.persistence.ButlerDataRef

visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional

Dataref to write visitCat for checkpoints

"""

bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))

for i, visit in enumerate(sorted(groupedDataRefs)):

# We don't use the bypasses since we need the psf info which does

# not have a bypass

# TODO: When DM-15500 is implemented in the Gen3 Butler, this

# can be fixed

# Do not read those that have already been read

if visitCat['used'][i]:

continue

if (i % self.config.nVisitsPerCheckpoint) == 0:

self.log.info("Retrieving metadata for %s %d (%d/%d)" %

(self.config.visitDataRefName, visit, i, len(groupedDataRefs)))

# Save checkpoint if desired

if visitCatDataRef is not None:

visitCatDataRef.put(visitCat)

# Note that the reference ccd is first in the list (if available).

# The first dataRef in the group will be the reference ccd (if available)

dataRef = groupedDataRefs[visit][0]

exp = dataRef.get(datasetType='calexp_sub', bbox=bbox,

flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)

visitInfo = exp.getInfo().getVisitInfo()

f = exp.getFilter()

psf = exp.getPsf()

rec = visitCat[i]

rec['visit'] = visit

rec['filtername'] = f.getName()

# TODO DM-26991: when gen2 is removed, gen3 workflow will make it

# much easier to get the wcs's necessary to recompute the pointing

# ra/dec at the center of the camera.

radec = visitInfo.getBoresightRaDec()

rec['telra'] = radec.getRa().asDegrees()

rec['teldec'] = radec.getDec().asDegrees()

rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()

rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()

rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)

rec['exptime'] = visitInfo.getExposureTime()

# convert from Pa to millibar

# Note that I don't know if this unit will need to be per-camera config

rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100

# Flag to signify if this is a "deep" field. Not currently used

rec['deepFlag'] = 0

# Relative flat scaling (1.0 means no relative scaling)

rec['scaling'][:] = 1.0

# Median delta aperture, to be measured from stars

rec['deltaAper'] = 0.0

rec['psfSigma'] = psf.computeShape().getDeterminantRadius()

if dataRef.datasetExists(datasetType='calexpBackground'):

# Get background for reference CCD

# This approximation is good enough for now

bgStats = (bg[0].getStatsImage().getImage().array

for bg in dataRef.get(datasetType='calexpBackground'))

rec['skyBackground'] = sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)

else:

self.log.warn('Sky background not found for visit %d / ccd %d' %

(visit, dataRef.dataId[self.config.ccdDataRefName]))

rec['skyBackground'] = -1.0

rec['used'] = 1

def _makeSourceMapper(self, sourceSchema):

"""

Make a schema mapper for fgcm sources

Parameters

----------

sourceSchema: `afwTable.Schema`

Default source schema from the butler

Returns

-------

sourceMapper: `afwTable.schemaMapper`

Mapper to the FGCM source schema

"""

# create a mapper to the preferred output

sourceMapper = afwTable.SchemaMapper(sourceSchema)

# map to ra/dec

sourceMapper.addMapping(sourceSchema['coord_ra'].asKey(), 'ra')

sourceMapper.addMapping(sourceSchema['coord_dec'].asKey(), 'dec')

sourceMapper.addMapping(sourceSchema['slot_Centroid_x'].asKey(), 'x')

sourceMapper.addMapping(sourceSchema['slot_Centroid_y'].asKey(), 'y')

# Add the mapping if the field exists in the input catalog.

# If the field does not exist, simply add it (set to False).

# This field is not required for calibration, but is useful

# to collate if available.

try:

sourceMapper.addMapping(sourceSchema[self.config.psfCandidateName].asKey(),

'psf_candidate')

except LookupError: