From 5bf5dc81a150d014ee64bd39e3ccb3ffd4382e8b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Nov 2026 11:36:21 +0100 Subject: [PATCH 01/54] GPU CMake: Mark GPUCA_DETERMINISTIC_MODE=1 invalid --- dependencies/FindO2GPU.cmake | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dependencies/FindO2GPU.cmake b/dependencies/FindO2GPU.cmake index 18472b0c19a40..a8a2924872bf0 100644 --- a/dependencies/FindO2GPU.cmake +++ b/dependencies/FindO2GPU.cmake @@ -10,7 +10,7 @@ # or submit itself to any jurisdiction. # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources -# FindO2GPU.cmake Version 24 +# FindO2GPU.cmake Version 25 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual) set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908) @@ -146,12 +146,13 @@ if(DEFINED ENV{O2_OVERRIDE_GPUCA_DETERMINISTIC_MODE}) set(GPUCA_DETERMINISTIC_MODE $ENV{O2_OVERRIDE_GPUCA_DETERMINISTIC_MODE}) endif() # set(GPUCA_DETERMINISTIC_MODE WHOLEO2) # Override -set(GPUCA_DETERMINISTIC_MODE_MAP_OFF 0) -set(GPUCA_DETERMINISTIC_MODE_MAP_NO_FAST_MATH 1) # No -ffast-math and similar compile flags for GPU folder -set(GPUCA_DETERMINISTIC_MODE_MAP_OPTO2 2) # In addition, -O2 optimization on host for GPU folder -set(GPUCA_DETERMINISTIC_MODE_MAP_GPU 3) # In addition, GPUCA_DETERMINISTIC_MODE define for GPU folder -set(GPUCA_DETERMINISTIC_MODE_MAP_ON 3) # Synonym for GPU -set(GPUCA_DETERMINISTIC_MODE_MAP_WHOLEO2 4) # As GPU but for whole O2 code +set(GPUCA_DETERMINISTIC_MODE_MAP_OFF 0) # Off - best performance +set(GPUCA_DETERMINISTIC_MODE_MAP_INVALID 1) # Invalid value, so people do not use 1 accidentally +set(GPUCA_DETERMINISTIC_MODE_MAP_NO_FAST_MATH 2) # No -ffast-math and similar compile flags for GPU folder +set(GPUCA_DETERMINISTIC_MODE_MAP_OPTO2 3) # In addition, -O2 optimization on host for GPU folder +set(GPUCA_DETERMINISTIC_MODE_MAP_GPU 4) # In addition, GPUCA_DETERMINISTIC_MODE define for GPU folder +set(GPUCA_DETERMINISTIC_MODE_MAP_ON 4) # Synonym for GPU +set(GPUCA_DETERMINISTIC_MODE_MAP_WHOLEO2 5) # As GPU but for whole O2 code if(NOT DEFINED GPUCA_DETERMINISTIC_MODE) set(GPUCA_DETERMINISTIC_MODE 0) elseif(NOT GPUCA_DETERMINISTIC_MODE MATCHES "^[0-9]+$") @@ -160,6 +161,9 @@ elseif(NOT GPUCA_DETERMINISTIC_MODE MATCHES "^[0-9]+$") endif() set(GPUCA_DETERMINISTIC_MODE ${GPUCA_DETERMINISTIC_MODE_MAP_${GPUCA_DETERMINISTIC_MODE}}) endif() +if(GPUCA_DETERMINISTIC_MODE STREQUAL 1) + message(FATAL_ERROR "GPUCA_DETERMINISTIC_MODE = 1 is invalid and not supported, use ON or GPU or see FindO2GPU.cmake for details") +endif() if(GPUCA_DETERMINISTIC_NO_FTZ) set(GPUCA_CXX_DENORMALS_FLAGS "") set(GPUCA_CUDA_DENORMALS_FLAGS "--ftz=false") From 54f6f25f802d4176524893dac0b4862cf97b0cac Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 15:52:07 +0200 Subject: [PATCH 02/54] GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 - .../Global/GPUChainTrackingMerger.cxx | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 -- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 49 +++++-------------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 7 files changed, 17 insertions(+), 53 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index e323f5f0fc6e5..6e963b8b06286 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -152,7 +152,6 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") -AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails (=2 means retry in same kernel, =1 for separate kernel") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 528a848acf389..fe1d06d27f602 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -235,9 +235,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); - if (param().rec.tpc.retryRefit == 1) { - runKernel(GetGridAuto(0), -1); - } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index cf8ea3c4f4675..233bdcf3e7d0b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,8 +402,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers - computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); // Reducing mNMaxTracks for mLoopData / mRetryRefitIds does not save memory, since the other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index eefaf6fb74497..a18ad20fc6e88 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -67,7 +67,6 @@ class GPUTPCGMMerger : public GPUProcessor static GPUglobalconstexpr() const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors struct memory { - GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -129,7 +128,6 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } - GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -299,7 +297,6 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; - uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index bf01ea234a645..a7df0d279c08f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,10 +21,9 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) { - const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { - const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, mode == -1); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { + const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7b70d94b9cfc5..74c9651d33dde 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) { constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -91,7 +91,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); - prop.SetSeedingErrors(!(refit && attempt == 0)); + prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); @@ -104,7 +104,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - int32_t goodRows = 0; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); if (crossCE) { @@ -178,11 +177,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (lastRow == 255 || CAMath::Abs((int32_t)lastRow - (int32_t)cluster.row) > 5 || lastSector != cluster.sector || (param.rec.tpc.trackFitRejectMode < 0 && -nMissed <= param.rec.tpc.trackFitRejectMode)) { - goodRows = 0; - } else { - goodRows++; - } if (retValProp == 0) { lastRow = cluster.row; lastSector = cluster.sector; @@ -227,17 +221,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { int8_t rejectChi2 = 0; - if (attempt == 0) { - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } else { - rejectChi2 = allowChangeClusters && goodRows > 5; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; } } @@ -904,7 +894,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, int32_t attempt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -919,27 +909,10 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, attempt, constants::MAX_SIN_PHI, track); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track); CADEBUG(printf("Finished Fit Track %d\n", iTrk)); CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); - if (!ok && attempt == 0 && merger->Param().rec.tpc.retryRefit) { - for (uint32_t i = 0; i < track.NClusters(); i++) { - merger->Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - CADEBUG(printf("Track rejected, marking for retry\n")); - if (merger->Param().rec.tpc.retryRefit == 2) { - nTrackHits = track.NClusters(); - NTolerated = 0; // Clusters not fit but tollerated for track length cut - t = track.Param(); - Alpha = track.Alpha(); - ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, 1, constants::MAX_SIN_PHI, track); - } else { - uint32_t nRefit = CAMath::AtomicAdd(&merger->Memory()->nRetryRefit, 1u); - merger->RetryRefitIds()[nRefit] = iTrk; - return; - } - } if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = 1.e-4f; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index c9d8f2017af52..fffa3f5180653 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, int32_t attempt, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,7 +200,7 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, int32_t attempt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); GPUdi() void ConstrainSinPhi(float limit = constants::MAX_SIN_PHI) { From 50ed2cd7b8eb998f3331d583495191eb305db932 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Sep 2025 21:34:57 +0200 Subject: [PATCH 03/54] GPU TPC: Add possibility to run last way of TPC fit in separate kernel to rebuild track ion between --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.cxx | 8 ++++ .../Global/GPUChainTrackingMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 46 +++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +- GPU/GPUTracking/kernels.cmake | 2 +- 8 files changed, 47 insertions(+), 26 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 6e963b8b06286..7146f1b7e3558 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -135,6 +135,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilon, uint8_t, 10, "", 0, "Cluster Finder: Dif AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression, relative as fraction of 255") AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") +AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 8a22e2c03ea47..8ab663b5f791a 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -266,6 +266,14 @@ bool GPUChainTracking::ValidateSettings() GPUError("Cannot do error interpolation with NWays < 3!"); return false; } + if (param().rec.tpc.rebuildTrackInFit && !param().rec.tpc.mergerInterpolateErrors) { + GPUError("Need error interpolation to rebuild tracks during fit"); + return false; + } + if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) { + GPUError("Need at least 1 cluster candidate to rebuild tracks during fit"); + return false; + } if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) { GPUError("configured max time bin exceeds 256 orbits"); return false; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index fe1d06d27f602..7b712831c2a23 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -234,7 +234,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); + if (param().rec.tpc.rebuildTrackInFit) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); + } runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index a7df0d279c08f..22d8b159e9b74 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -19,11 +19,11 @@ using namespace o2::gpu; template <> -GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode) +GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index d703f1b890443..b4f823aa11ed8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -36,7 +36,7 @@ class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral { public: template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt); }; class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 74c9651d33dde..f1abf9ecd866d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float kDeg2Rad = M_PI / 180.f; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); @@ -60,7 +60,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetMaterialTPC(); prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); - if (param.rec.tpc.mergerInterpolateErrors) { + if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { for (int32_t i = 0; i < N; i++) { interpolation.hit[i].errorY = -1; } @@ -70,18 +70,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const int32_t maxN = N; int32_t ihitStart = 0; float covYYUpd = 0.f; - float lastUpdateX = -1.f; - uint8_t lastRow = 255; - uint8_t lastSector = 255; float deltaZ = 0.f; - for (int32_t iWay = 0; iWay < nWays; iWay++) { + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; int32_t nAvgCharge = 0; if (iWay && (iWay & 1) == 0) { - StoreOuter(&track.OuterParam(), prop.GetAlpha()); + StoreOuter(&track.OuterParam(), prop.GetAlpha()); // TODO: Fix outer param! } int32_t resetT0 = initResetT0(); @@ -95,12 +92,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); - prop.SetTrack(this, iWay ? prop.GetAlpha() : Alpha); + prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - lastUpdateX = -1; + uint8_t lastRow = 255; + uint8_t lastSector = 255; + float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -110,9 +109,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ lastSector = clusters[ihit].sector; } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (finalOutInFit && !(clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject)) { + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; } continue; @@ -327,6 +329,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } else { deltaZ = 0.f; } + + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + Alpha = prop.GetAlpha(); + if (ihitStart != 0) { + MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); + } + return true; + } } ConstrainSinPhi(); @@ -894,7 +904,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -908,21 +918,21 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - CADEBUG(int32_t nTrackHitsOld = nTrackHits; float ptOld = t.QPt()); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track); - CADEBUG(printf("Finished Fit Track %d\n", iTrk)); - CADEBUG(printf("OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", nTrackHitsOld, nTrackHits, NTolerated, nTrackHits + NTolerated, ptOld, t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track, rebuilt); + CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { - t.QPt() = 1.e-4f; + t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); track.SetOK(ok); - track.SetNClustersFitted(nTrackHits); track.Param() = t; track.Alpha() = Alpha; + if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + track.SetNClustersFitted(nTrackHits); + } // if (track.OK()) merger->DebugRefitMergedTrack(track); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index fffa3f5180653..9a9b22bbd2f7a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,8 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); @@ -200,8 +201,6 @@ class GPUTPCGMTrackParam } } - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger); - GPUdi() void ConstrainSinPhi(float limit = constants::MAX_SIN_PHI) { if (mP[2] > limit) { diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index ed155788b0bef..138eb8a6339db 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -51,7 +51,7 @@ o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, mergedTracks2" "= TPC o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, borderTracks" "= TPCMERGER" NO int8_t parameter) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fill" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fold" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map uint32_t* output) -o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode int32_t rebuilt) o2_gpu_add_kernel("GPUTPCGMMergerFollowLoopers" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT" LB) o2_gpu_add_kernel("GPUTPCGMMergerUnpackResetIds" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iSector) o2_gpu_add_kernel("GPUTPCGMMergerSectorRefit" "GPUTPCGMMergerGPU TPCMERGER MATLUT" LB int32_t iSector) From b1989234c922714b220486030b89d32f6afe0e15 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 23 Sep 2025 17:58:33 +0200 Subject: [PATCH 04/54] GPU TPC: Search hit closest to interpolated point in track rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 26 ++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 131 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 1 + .../GPUTPCTrackletConstructor.cxx | 4 +- 8 files changed, 149 insertions(+), 26 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 7146f1b7e3558..25490bde02f18 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -136,6 +136,7 @@ AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Fin AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") +AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 7b712831c2a23..d22778ea41b63 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -96,7 +96,6 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) uint32_t numBlocks = (!mRec->IsGPU() || doGPU) ? BlockCount() : 1; GPUTPCGMMerger& Merger = processors()->tpcMerger; GPUTPCGMMerger& MergerShadow = doGPU ? processorsShadow()->tpcMerger : Merger; - GPUTPCGMMerger& MergerShadowAll = doGPU ? processorsShadow()->tpcMerger : Merger; const int32_t outputStream = OutputStream(); if (GetProcessingSettings().debugLevel >= 2) { GPUInfo("Running TPC Merger"); @@ -142,7 +141,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) // Merge within Sectors runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeWithinSector, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); @@ -150,23 +149,23 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) // Merge between sectors - transport to the middle of the sector and rotate vertically to the border on the left / right runKernel(GetGridAuto(0, deviceType), false); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 2, 3, 0); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector | GPUTPCGMMerger::mergeModes::mergeAtMidRow, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); // Merge between sectors - transport to the left / right edge of the sector and rotate horizontally - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 0); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); // Merge between sectors - use original track param - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); runKernel(GetGridBlk(std::max(2u, numBlocks), 0, deviceType), 0, 1, 1); RunTPCTrackingMerger_MergeBorderTracks(GPUTPCGMMerger::mergeModes::mergeBetweenSector | GPUTPCGMMerger::mergeModes::mergeWithOriginalParameters, deviceType); RunTPCTrackingMerger_Resolve(0, 1, deviceType); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingMatching, doGPU, Merger, &GPUTPCGMMerger::DumpMergedBetweenSectors, *mDebugFile); - runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadowAll.TmpCounter())); + runKernel({{1, -WarpSize(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TmpCounter(), 2 * NSECTORS * sizeof(*MergerShadow.TmpCounter())); runKernel(GetGridAuto(0, deviceType)); if (GetProcessingSettings().mergerSanityCheck) { @@ -208,8 +207,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (maxId > Merger.NMaxClusters()) { throw std::runtime_error("mNMaxClusters too small"); } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.SharedCount(), maxId * sizeof(*MergerShadowAll.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadowAll.ClusterAttachment(), maxId * sizeof(*MergerShadowAll.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -234,6 +233,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) mOutputQueue.clear(); } + if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); @@ -268,13 +270,13 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) throw std::runtime_error("QA Scratch buffer exceeded"); } } - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadowAll.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracks(), MergerShadow.MergedTracks(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracks()), outputStream, 0, nullptr, waitEvent); waitEvent = nullptr; if (param().dodEdxEnabled) { - GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadowAll.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } - GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadowAll.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadowAll.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 233bdcf3e7d0b..9b7e2c0256f8a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -403,6 +403,7 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -1656,7 +1657,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; } nHits += nTrackHits; } @@ -1964,7 +1965,7 @@ GPUd() void GPUTPCGMMerger::MergeLoopersInit(int32_t nBlocks, int32_t nThreads, for (uint32_t k = 0;k < trk.NClusters();k++) { float xx, yy, zz; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[mClusters[trk.FirstClusterRef() + k].num]; - GetConstantMem()->calibObjects.fastTransformHelper->Transform(mClusters[trk.FirstClusterRef() + k].sector, mClusters[trk.FirstClusterRef() + k].row, cl.getPad(), cl.getTime(), xx, yy, zz, p.GetTOffset()); + GetConstantMem()->calibObjects.fastTransform->Transform(mClusters[trk.FirstClusterRef() + k].sector, mClusters[trk.FirstClusterRef() + k].row, cl.getPad(), cl.getTime(), xx, yy, zz, p.GetTOffset()); float sa2, ca2; CAMath::SinCos(Param().Alpha(mClusters[trk.FirstClusterRef() + k].sector), sa2, ca2); float cx = ca2 * xx - sa2 * yy; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index a18ad20fc6e88..2afca0512783d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -78,10 +78,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nLooperMatchCandidates; }; - struct trackCluster { + struct trackCluster { // TODO: Reduce size of this struct! uint32_t id; uint8_t row; uint8_t sector; + float error; }; struct tmpSort { @@ -125,6 +126,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } + GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -272,6 +274,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + trackCluster* mClusterCandidates = nullptr; GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index f1abf9ecd866d..57a2de5979fb1 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,9 +47,10 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, float maxSinPhi, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float kDeg2Rad = M_PI / 180.f; + constexpr float maxSinPhi = constants::MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); const GPUParam& GPUrestrict() param = merger->Param(); @@ -72,9 +73,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; - for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // DR: Unrolling has no performance improvement on GPU, why? + for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; - float sumInvSqrtCharge = 0.f; + float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; if (iWay && (iWay & 1) == 0) { @@ -135,7 +136,26 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, allowChangeClusters) == -1) { + if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { + while (true) { + if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { + break; + } + } + const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (interHit.id == 1) { + nMissed++; + nMissed2++; + continue; + } + const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; + merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (interHit.id - 2 == clusters[ihit].num) { + clusterState = clusters[ihit].state; + } else { + clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -155,7 +175,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // clang-format off CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); // clang-format on - if (allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, constants::MAX_SIN_PHI, dodEdx); @@ -200,7 +220,102 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } float uncorrectedY = -1e6f; - if (allowChangeClusters) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); + const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); + GPUglobalref() const cahit2* hits = tracker.HitData(row); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); + const auto& inter = interpolation.hit[ihit]; + if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float y0 = row.Grid().YMin(); + const float stepY = row.HstepY(); + const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = row.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + float uncorrectedZ; + merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = row.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + } + if (nCandidates == 0) { + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } @@ -320,7 +435,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ break; // bad chi2 for the whole track, stop the fit } } - if (finalOutInFit && !(merger->Param().rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -918,7 +1033,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, constants::MAX_SIN_PHI, track, rebuilt); + bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 9a9b22bbd2f7a..b287e5a717edf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -141,7 +141,7 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, float maxSinPhi, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index cb0c8d4a76fdb..7c9047202417b 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -167,6 +167,7 @@ class GPUTPCTracker : public GPUProcessor } return ((int32_t)weight); } + GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } GPUd() void MaximizeHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.MaximizeHitWeight(row, hitIndex, weight); } GPUd() void SetHitWeight(const GPUTPCRow& row, int32_t hitIndex, int32_t weight) { mData.SetHitWeight(row, hitIndex, weight); } GPUd() int32_t HitWeight(const GPUTPCRow& row, int32_t hitIndex) const { return mData.HitWeight(row, hitIndex); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 33a3264a87ab3..86979c29006ae 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -199,7 +199,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float sErr2 = tracker.Param().GetSystematicClusterErrorIFC2(x, tParam.GetY(), tParam.GetZ(), tracker.ISector() >= 18); err2Y += sErr2; err2Z += sErr2; - const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; + const float kFactor = tracker.GetChiSeedFactor(); float sy2 = kFactor * (tParam.Err2Y() + err2Y); float sz2 = kFactor * (tParam.Err2Z() + err2Z); if (sy2 > tracker.Param().rec.tpc.hitSearchArea2) { @@ -307,7 +307,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 7.0f * 7.0f; const float maxWindow2 = tracker.Param().rec.tpc.hitSearchArea2; const float sy2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Y() + err2Y)); - const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); + const float sz2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Z() + err2Z)); // TODO: Double check if the factors are enough, errors could be larger with flags int32_t bin, ny, nz; row.Grid().GetBinArea(yUncorrected, zUncorrected + tParam.ZOffset(), CAMath::Sqrt(sy2), CAMath::Sqrt(sz2), bin, ny, nz); From db6586f1a23f75edd4855b6f0b3e678d1494ff5b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 5 Oct 2025 18:31:00 +0200 Subject: [PATCH 05/54] GPU TPC: Fix index for hit interpolation in rows with multiple clusters --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 57a2de5979fb1..c5484cc8f523c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -219,13 +219,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - const auto& inter = interpolation.hit[ihit]; if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); const float y0 = row.Grid().YMin(); @@ -357,10 +357,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && interpolation.hit[ihit].errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { rejectChi2 = GPUTPCGMPropagator::rejectDirect; } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &interpolation.hit[ihit], err2Y, err2Z, deltaZ); + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); } } @@ -370,7 +370,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); } GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, interpolation.hit[ihit], rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); }); } // clang-format off From 6590cdba1a85b45636277103d568bffb19642480 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 29 Sep 2025 22:01:44 +0200 Subject: [PATCH 06/54] GPU TPC: Implement ambiguity solving for rebuild tracks cluster association --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- .../Definitions/Parameters/GPUParameters.csv | 5 + .../GPUChainTrackingDebugAndProfiling.cxx | 6 +- .../Global/GPUChainTrackingMerger.cxx | 22 +- GPU/GPUTracking/Global/GPUErrorCodes.h | 20 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 220 ++++++++++++++++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 27 ++- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 31 +++ GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 13 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +-- GPU/GPUTracking/SectorTracker/GPUTPCTracker.h | 5 +- .../Standalone/Benchmark/standalone.cxx | 2 +- GPU/GPUTracking/kernels.cmake | 5 + GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 15 files changed, 322 insertions(+), 79 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 25490bde02f18..f14a757d63970 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -137,12 +137,13 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following, 8: mirroring)") +AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 8e41173cee40a..5892f4acfe9c4 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -52,6 +52,11 @@ GPUTPCGMMergerPrepareForFit_step2,256,,256,256,,,,,,256,256,256,256,,256,,256,25 GPUTPCGMMergerFinalize_step0,256,,,256,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerFinalize_step1,256,,,256,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerFinalize_step2,256,,,256,,,,,,256,,256,256,,256,,256,256, +GPUTPCGMMergerHitWeights_prepare,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_compute,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_resolve1,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_resolve2,256,,,,,,,,,,,,,,,,,, +GPUTPCGMMergerHitWeights_resolveShared,256,,,,,,,,,,,,,,,,,, GPUTPCGMMergerMergeLoopers_step0,256,,,,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerMergeLoopers_step1,256,,,,,,,,,256,,256,256,,256,,256,256, GPUTPCGMMergerMergeLoopers_step2,256,,,,,,,,,256,,256,256,,256,,256,256, diff --git a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx index 7659d19693777..e45aeb4ba9246 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx @@ -187,8 +187,8 @@ void GPUChainTracking::PrintMemoryRelations() GPUInfo("MEMREL SectorTrackHits NCl %d NTrkH %d", processors()->tpcTrackers[i].NHitsTotal(), *processors()->tpcTrackers[i].NTrackHits()); } if (processors()->tpcMerger.Memory()) { - GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTracks()); - GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NMaxClusters(), processors()->tpcMerger.NMergedTrackClusters()); + GPUInfo("MEMREL Tracks NCl %d NTrk %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTracks()); + GPUInfo("MEMREL TrackHitss NCl %d NTrkH %d", processors()->tpcMerger.NClusters(), processors()->tpcMerger.NMergedTrackClusters()); } } @@ -217,7 +217,7 @@ void GPUChainTracking::PrintKernelDebugOutput() void GPUChainTracking::PrintOutputStat() { int32_t nTracks = 0, nAttachedClusters = 0, nAttachedClustersFitted = 0, nAdjacentClusters = 0; - uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NMaxClusters(); + uint32_t nCls = GetProcessingSettings().doublePipeline ? mIOPtrs.clustersNative->nClustersTotal : processors()->tpcMerger.NClusters(); if (GetProcessingSettings().createO2Output > 1) { nTracks = mIOPtrs.nOutputTracksTPCO2; nAttachedClusters = mIOPtrs.nMergedTrackHits; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index d22778ea41b63..7f393446b2744 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -203,12 +203,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) Merger.CheckCollectedTracks(); } - uint32_t maxId = Merger.NMaxClusters(); - if (maxId > Merger.NMaxClusters()) { - throw std::runtime_error("mNMaxClusters too small"); - } - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), maxId * sizeof(*MergerShadow.SharedCount())); - runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), maxId * sizeof(*MergerShadow.ClusterAttachment())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.SharedCount(), Merger.NClusters() * sizeof(*MergerShadow.SharedCount())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*MergerShadow.ClusterAttachment())); runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); runKernel(GetGridAuto(0, deviceType)); @@ -238,6 +234,16 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + for (int32_t i = 0; i < param().rec.tpc.rebuildTrackInFitClusterCandidates; i++) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + if (i + 1 < param().rec.tpc.rebuildTrackInFitClusterCandidates) { + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), i); + } + } + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); @@ -276,7 +282,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) GPUMemCpy(RecoStep::TPCMerging, Merger.MergedTracksdEdx(), MergerShadow.MergedTracksdEdx(), Merger.NMergedTracks() * sizeof(*Merger.MergedTracksdEdx()), outputStream, 0); } GPUMemCpy(RecoStep::TPCMerging, Merger.Clusters(), MergerShadow.Clusters(), Merger.NMergedTrackClusters() * sizeof(*Merger.Clusters()), outputStream, 0); - GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NMaxClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); + GPUMemCpy(RecoStep::TPCMerging, Merger.ClusterAttachment(), MergerShadow.ClusterAttachment(), Merger.NClusters() * sizeof(*Merger.ClusterAttachment()), outputStream, 0); } if (GetProcessingSettings().outputSharedClusterMap) { TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutputState(), outputStream, nullptr, waitEvent); @@ -366,7 +372,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (GetProcessingSettings().debugLevel >= 2) { - GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NClusters()); + GPUInfo("TPC Merger Finished (output clusters %d / input clusters %d)", Merger.NMergedTrackClusters(), Merger.NSectorHits()); } return 0; } diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index a4921f478b107..2080548c7775c 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -33,16 +33,16 @@ GPUCA_ERROR_CODE(13, ERROR_SECTORDATA_HITINROW_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(14, ERROR_SECTORDATA_BIN_OVERFLOW, SectorRow, Value, Max) GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(17, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(18, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER - +GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 9b7e2c0256f8a..d9b139b73ab7e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -398,12 +398,17 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackIDs, GPUTPCGeometry::NSECTORS * mNMaxSingleSectorTracks); // UnpackResetIds - RefitSectorTracks - UnpackSectorGlobal memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 - computePointerWithAlignment(mem, mSharedCount, mNMaxClusters); + computePointerWithAlignment(mem, mSharedCount, mNClusters); // PrepareForFit0 - SortTracksQPt - PrepareForFit1 - PrepareForFit1 / Finalize0 - Finalize2 + computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway - computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { + computePointerWithAlignment(mem, mSharedCount2, mNClusters); + computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); + computePointerWithAlignment(mem, mHitWeights, mNClusters); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 @@ -437,14 +442,14 @@ void* GPUTPCGMMerger::SetPointersOutput(void* mem) } } computePointerWithAlignment(mem, mClusters, mNMaxMergedTrackClusters); - computePointerWithAlignment(mem, mClusterAttachment, mNMaxClusters); + computePointerWithAlignment(mem, mClusterAttachment, mNClusters); return mem; } void* GPUTPCGMMerger::SetPointersOutputState(void* mem) { if ((mRec->GetRecoSteps() & gpudatatypes::RecoStep::Refit) || mRec->GetProcessingSettings().outputSharedClusterMap) { - computePointerWithAlignment(mem, mClusterStateExt, mNMaxClusters); + computePointerWithAlignment(mem, mClusterStateExt, mNClusters); } else { mClusterStateExt = nullptr; } @@ -497,33 +502,33 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() void GPUTPCGMMerger::SetMaxData(const GPUTrackingInOutPointers& io) { mNTotalSectorTracks = 0; - mNClusters = 0; + mNSectorHits = 0; mNMaxSingleSectorTracks = 0; for (int32_t iSector = 0; iSector < NSECTORS; iSector++) { uint32_t ntrk = *mRec->GetConstantMem().tpcTrackers[iSector].NTracks(); mNTotalSectorTracks += ntrk; - mNClusters += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); + mNSectorHits += *mRec->GetConstantMem().tpcTrackers[iSector].NTrackHits(); if (mNMaxSingleSectorTracks < ntrk) { mNMaxSingleSectorTracks = ntrk; } } - mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNClusters); + mNMaxMergedTrackClusters = mRec->MemoryScalers()->NTPCMergedTrackHits(mNSectorHits); if (CAMath::Abs(Param().polynomialField.GetNominalBz()) < (gpu_common_constants::kZeroFieldCut * gpu_common_constants::kCLight)) { mNMaxTracks = mRec->MemoryScalers()->getValue(mNTotalSectorTracks, mNTotalSectorTracks); // 0 magnetic field } else { mNMaxTracks = mRec->MemoryScalers()->NTPCMergedTracks(mNTotalSectorTracks); } if (io.clustersNative) { - mNMaxClusters = io.clustersNative->nClustersTotal; + mNClusters = io.clustersNative->nClustersTotal; } else if (mRec->GetRecoSteps() & gpudatatypes::RecoStep::TPCSectorTracking) { - mNMaxClusters = 0; + mNClusters = 0; for (int32_t i = 0; i < NSECTORS; i++) { - mNMaxClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); + mNClusters += mRec->GetConstantMem().tpcTrackers[i].NHitsTotal(); } } else { - mNMaxClusters = mNClusters; + mNClusters = mNSectorHits; } - mNMaxLooperMatches = mNMaxClusters / 4; // We have that much scratch memory anyway + mNMaxLooperMatches = mNClusters / 4; // We have that much scratch memory anyway } int32_t GPUTPCGMMerger::CheckSectors() @@ -1657,7 +1662,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f}; + *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0}; } nHits += nTrackHits; } @@ -1838,11 +1843,13 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { for (uint32_t j = 0; j < trk.NClusters(); j++) { - uint32_t weight = attachAttached | attachGood; - if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { - weight |= attachProtect; + if (!Param().rec.tpc.rebuildTrackInFit) { + uint32_t weight = attachAttached | attachGood; + if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { + weight |= attachProtect; + } + CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); } - CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } if (!trk.CCE() && !trk.MergedLooper()) { @@ -1873,8 +1880,8 @@ GPUd() void GPUTPCGMMerger::PrepareForFit2(int32_t nBlocks, int32_t nThreads, in mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; } } - if (mClusterStateExt) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nBlocks * nThreads) { + if (!Param().rec.tpc.rebuildTrackInFit && mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); if (mSharedCount[i] > 1) { state |= GPUTPCGMMergedTrackHit::flagShared; @@ -1925,7 +1932,7 @@ GPUd() void GPUTPCGMMerger::Finalize1(int32_t nBlocks, int32_t nThreads, int32_t GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { - for (uint32_t i = iBlock * nThreads + iThread; i < mNMaxClusters; i += nThreads * nBlocks) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nThreads * nBlocks) { if (mClusterAttachment[i] != 0) { mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | mTrackSort[mClusterAttachment[i] & attachTrackMask]; } @@ -2120,3 +2127,172 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, } #endif*/ } + +GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + mSharedCount2[i] = 0; + } + if (iThread == 0 && iBlock == 0) { + mMemory->nMergedTrackClusters = 0; + } + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } + } + } +} + +GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].id >= 2) { + const int32_t nCl = trk.NClusters(); + float weightf = nCl * (nCl * 2 - 5) * 128 / trk.GetParam().GetChi2() / (1.f + candidates[iteration].error); + int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); + candidates[iteration].weight = weight; + CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); + CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); + } else { + candidates[0].best = -1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration) +{ + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + continue; + } + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best == 0) { + if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) { + candidates[0].best = iteration + 1; + } + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + int32_t nIter = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + trk.SetNClusters(0); + continue; + } + uint32_t nHits = 0; + uint8_t sharedRowCandidates[GPUTPCGeometry::NROWS]; + uint8_t nSharedRowCandidates = 0; + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (trk.GetParam().GetNDF() > 0 && candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { + candidates[0].best = nIter; + } + if (candidates[0].best > 0) { + nHits++; + if (candidates[0].best != 128 && candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { + mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations + } + } else if (candidates[0].best == 0 && candidates[0].id >= 2) { + sharedRowCandidates[nSharedRowCandidates++] = j; + } + } + + if (nSharedRowCandidates) { + const uint32_t nShared = CAMath::Min(nSharedRowCandidates, CAMath::Float2UIntRn(nHits * Param().rec.tpc.rebuildTrackMaxSharedFraction)); + if (nShared) { + const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; + const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n]; + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) { + const auto& ca = candidates[a * n]; + const auto& cb = candidates[b * n]; + return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error); + }); + for (uint32_t j = 0; j < nShared; j++) { + auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error)); + cj[0].best = 1; + mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters? + } + nHits += nShared; + } + } + + const uint32_t iOut = CAMath::AtomicAdd(&mMemory->nMergedTrackClusters, nHits); + if (iOut + nHits > mNMaxMergedTrackClusters) { + raiseError(GPUErrors::ERROR_MERGER_REBUILD_HIT_OVERFLOW, iOut, mNMaxMergedTrackClusters); + CAMath::AtomicExch(&mMemory->nMergedTrackClusters, mNMaxMergedTrackClusters); + trk.SetNClusters(0); + trk.SetOK(false); + continue; + } + + trk.SetNClusters(nHits); + trk.SetFirstClusterRef(iOut); + uint32_t written = 0; + const bool reverse = mTrackRebuildHelper[i].reverse; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut + for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { + const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].best > 0) { + auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; + if (candidates[0].best == 128) { + outCl = {.num = candidates[0].id - 2, .sector = candidates[0].sector, .row = (uint8_t)j, .state = (uint8_t)candidates[0].weight}; + written++; + continue; + } + const auto& best = candidates[candidates[0].best - 1]; + const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + written++; + CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); + } + } + } +} + +GPUd() void GPUTPCGMMerger::ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTrackClusters; i += nBlocks * nThreads) { + if (mSharedCount2[mClusters[i].num]) { + mClusters[i].state |= GPUTPCGMMergedTrackHit::flagShared; // TODO: What is the best criterion to set the shared flag? Perhaps already when iter0 best hit would be shared? + } + } + if (mClusterStateExt) { + for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { + uint8_t state = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[i].getFlags(); + if (mSharedCount2[i]) { + state |= GPUTPCGMMergedTrackHit::flagShared; + } + mClusterStateExt[i] = state; + } + } +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 2afca0512783d..30c76aa2af810 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -83,6 +83,12 @@ class GPUTPCGMMerger : public GPUProcessor uint8_t row; uint8_t sector; float error; + int32_t weight; + int32_t best; + }; + + struct trackRebuildHelper { + bool reverse; }; struct tmpSort { @@ -119,14 +125,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdxAlt() const { return mMergedTracksdEdxAlt; } GPUhdi() GPUdEdxInfo* MergedTracksdEdxAlt() { return mMergedTracksdEdxAlt; } + GPUhdi() uint32_t NSectorHits() const { return mNSectorHits; } GPUhdi() uint32_t NClusters() const { return mNClusters; } - GPUhdi() uint32_t NMaxClusters() const { return mNMaxClusters; } GPUhdi() uint32_t NMaxTracks() const { return mNMaxTracks; } GPUhdi() uint32_t NMaxMergedTrackClusters() const { return mNMaxMergedTrackClusters; } GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } - GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return (mClusters); } - GPUhdi() trackCluster* ClusterCandidates() { return (mClusterCandidates); } + GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } + GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } @@ -139,6 +146,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() tmpSort* TrackSortO2() { return mTrackSortO2; } GPUhdi() internal::MergeLooperParam* LooperCandidates() { return mLooperCandidates; } GPUhdi() GPUAtomic(uint32_t) * SharedCount() { return mSharedCount; } + GPUhdi() uint8_t* SharedCount2() { return mSharedCount2; } GPUhdi() gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) { return mBorderRange[i]; } GPUhdi() const gputpcgmmergertypes::GPUTPCGMBorderRange* BorderRange(int32_t i) const { return mBorderRange[i]; } GPUhdi() GPUTPCGMBorderTrack* BorderTracks(int32_t i) { return mBorder[i]; } @@ -196,6 +204,11 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersSort(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersMain(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); + GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -261,7 +274,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t mNMaxTracks = 0; // maximum number of output tracks uint32_t mNMaxSingleSectorTracks = 0; // max N tracks in one sector uint32_t mNMaxMergedTrackClusters = 0; // max number of clusters in output tracks (double-counting shared clusters) - uint32_t mNMaxClusters = 0; // max total unique clusters (in event) + uint32_t mNClusters = 0; // max total unique clusters (in event) uint32_t mNMaxLooperMatches = 0; // Maximum number of candidate pairs for looper matching uint16_t mMemoryResMemory = (uint16_t)-1; @@ -272,9 +285,12 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; - int32_t mNClusters = 0; // Total number of incoming clusters (from sector tracks) + int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks trackCluster* mClusterCandidates = nullptr; + trackRebuildHelper* mTrackRebuildHelper = nullptr; + int32_t* mHitWeights = nullptr; + GPUdEdxInfo* mMergedTracksdEdx = nullptr; //* dEdx information GPUdEdxInfo* mMergedTracksdEdxAlt = nullptr; //* dEdx alternative information GPUTPCGMSectorTrack* mSectorTrackInfos = nullptr; //* additional information for sector tracks @@ -295,6 +311,7 @@ class GPUTPCGMMerger : public GPUProcessor uint32_t* mTrackSort = nullptr; tmpSort* mTrackSortO2 = nullptr; GPUAtomic(uint32_t) * mSharedCount = nullptr; // Must be uint32_t unfortunately for atomic support + uint8_t* mSharedCount2 = nullptr; GPUTPCGMBorderTrack* mBorderMemory = nullptr; // memory for border tracks GPUTPCGMBorderTrack* mBorder[2 * NSECTORS]; gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 9f6858df4d3f2..c9801162cef40 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -192,7 +192,7 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const out << " Track Clusters"; DumpTrackClusters(out); uint32_t j = 0; - for (uint32_t i = 0; i < mNMaxClusters; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (j++ % 10 == 0) { out << "\n Cluster attachment "; @@ -241,7 +241,7 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "Track Clusters\n"; DumpTrackClusters(out, true); uint32_t j = 0; - for (uint32_t i = 0; i < mNMaxClusters; i++) { + for (uint32_t i = 0; i < mNClusters; i++) { if ((mClusterAttachment[i] & attachFlagMask) != 0) { if (++j % 10 == 0) { out << " Cluster attachment "; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 22d8b159e9b74..d3ea60b4dc099 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -214,3 +214,34 @@ GPUdii() void GPUTPCGMMergerMergeLoopers::Thread<2>(int32_t nBlocks, int32_t nTh { merger.MergeLoopersMain(nBlocks, nThreads, iBlock, iThread); } + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.PrepareHitWeights(nBlocks, nThreads, iBlock, iThread); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ComputeHitWeights(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights1(nBlocks, nThreads, iBlock, iThread, iteration); +} + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeights2(nBlocks, nThreads, iBlock, iThread); +} + + +template <> +GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) +{ + merger.ResolveHitWeightsShared(nBlocks, nThreads, iBlock, iThread); +} diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index b4f823aa11ed8..82605f47a5f27 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -180,6 +180,19 @@ class GPUTPCGMMergerMergeLoopers : public GPUTPCGMMergerGeneral GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); }; +class GPUTPCGMMergerHitWeights : public GPUTPCGMMergerGeneral +{ + public: + enum K { defaultKernel = 0, + prepare = 0, + compute = 1, + resolve1 = 2, + resolve2 = 3, + resolveShared = 4 }; + template + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t iteration); +}; + } // namespace o2::gpu #endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index c5484cc8f523c..42b65abeca828 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -136,26 +136,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on - if (rebuilt && merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id > 0) { - while (true) { - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - } - const auto& interHit = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + clusters[ihit].row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - if (interHit.id == 1) { - nMissed++; - nMissed2++; - continue; - } - const ClusterNative& GPUrestrict() cl = merger -> GetConstantMem()->ioPtrs.clustersNative->clustersLinear[interHit.id - 2]; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - if (interHit.id - 2 == clusters[ihit].num) { - clusterState = clusters[ihit].state; - } else { - clusterState = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - } - } else if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; @@ -227,7 +208,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin); + const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -303,13 +284,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; } - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err}; + merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } + CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -1042,9 +1024,13 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); - track.Param() = t; - track.Alpha() = Alpha; + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.Param().NDF() = 0; + } else { + track.Param() = t; + track.Alpha() = Alpha; + } if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h index 7c9047202417b..aaf1bff882522 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTracker.h @@ -162,9 +162,12 @@ class GPUTPCTracker : public GPUProcessor GPUdi() static int32_t CalculateHitWeight(int32_t NHits, float chi2) { float weight = NHits * (NHits * 2 - 5) * 128 / chi2; // TODO: Add QPt to this formula - if (weight < 0.f || weight > 2e9f) { + if (weight < 0.f) { return 0; } + if (weight > 2e9f) { + return 2e9f; + } return ((int32_t)weight); } GPUdi() float GetChiSeedFactor() const { return Param().rec.tpc.hitPickUpFactor * Param().rec.tpc.hitPickUpFactor * 3.5f * 3.5f; } diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 1b2403af07733..38ca9adb3d242 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -996,7 +996,7 @@ int32_t main(int argc, char** argv) nEventsProcessed++; if (configStandalone.timeFrameTime) { - double nClusters = chainTracking->GetProcessors()->tpcMerger.NMaxClusters(); + double nClusters = chainTracking->GetProcessors()->tpcMerger.NClusters(); if (nClusters > 0) { const int32_t nOrbits = 32; const double colRate = 50000; diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 138eb8a6339db..539840baaff7a 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -84,6 +84,11 @@ o2_gpu_add_kernel("GPUTPCGMMergerFinalize, step2" "GPUTP o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step0" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step1" "GPUTPCGMMergerGPU TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMMergerMergeLoopers, step2" "GPUTPCGMMergerGPU TPCMERGER" LB) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, prepare" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, compute" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve1" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolve2" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) +o2_gpu_add_kernel("GPUTPCGMMergerHitWeights, resolveShared" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iteration) o2_gpu_add_kernel("GPUTPCGMO2Output, prepare" "= TPCMERGER" LB) o2_gpu_add_kernel("GPUTPCGMO2Output, sort" "= TPCMERGER") o2_gpu_add_kernel("GPUTPCGMO2Output, output" "= TPCMERGER" LB) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 86aaea47b57f0..689f9a36fc2fe 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -1870,7 +1870,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } } - uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NMaxClusters(); + uint32_t nCl = clNative ? clNative->nClustersTotal : mTracking->GetProcessors()->tpcMerger.NClusters(); mClusterCounts.nTotal += nCl; if (mQATasks & (taskClusterCounts | taskClusterRejection)) { for (uint32_t iSector = 0; iSector < GPUTPCGeometry::NSECTORS; iSector++) { From 33ee65cd5ce396a7193ae5797df1db9d2c22d909 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Feb 2026 20:39:07 +0100 Subject: [PATCH 07/54] GPU: Additional debug dumps --- .../Global/GPUChainTrackingMerger.cxx | 2 + GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 37 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 7f393446b2744..52cdcd5dee78f 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -243,7 +243,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } } runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); + DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 30c76aa2af810..d07a47d4fe981 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -222,6 +222,8 @@ class GPUTPCGMMerger : public GPUProcessor void DumpRefit(std::ostream& out) const; void DumpFinal(std::ostream& out) const; void DumpLoopers(std::ostream& out) const; + void DumpInterpolatedHits(std::ostream& out) const; + void DumpRebuiltTracks(std::ostream& out) const; void DumpTrackParam(std::ostream& out) const; void DumpTrackClusters(std::ostream& out, bool non0StateOnly = false, bool noNDF0 = false) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index c9801162cef40..d22fd2e05e0a0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -150,6 +150,15 @@ void GPUTPCGMMerger::DumpTrackParam(std::ostream& out) const out << std::setprecision(ss); } +void GPUTPCGMMerger::DumpRebuiltTracks(std::ostream& out) const +{ + out << "\nTPC Merger Rebuilt Tracks\n"; + out << " Cluster Attachment\n"; + DumpTrackClusters(out, false, true); + out << " Track Params\n"; + DumpTrackParam(out); +} + void GPUTPCGMMerger::DumpMergeCE(std::ostream& out) const { DumpTrackLinks(out, true, " for CE merging"); @@ -255,6 +264,34 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const out << "\n"; } +void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const +{ + out << "\nTPC Merger Interpolated Hits\n"; + for (uint32_t i = 0; i < mMemory->nMergedTracks; i++) { + const auto& trk = mMergedTracks[i]; + if (trk.OK() && trk.GetParam().GetNDF() >= 0 && trk.NClusters()) { + out << "Track " << i << ":"; + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidates[0].id) { + out << " Row " << j << ": "; + if (candidates[0].best) { + out << " Best " << candidates[0].best << " - "; + } + for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { + if (candidates[k].id) { + out << k << ": id " << candidates[k].id << " err " << candidates[k].error << " weight " << candidates[k].weight << " - "; + } + } + out << "; "; + } + } + out << "\n"; + } + } + out << "\n"; +} + template inline void GPUTPCGMMerger::MergedTrackStreamerInternal(const GPUTPCGMBorderTrack& b1, const GPUTPCGMBorderTrack& b2, const char* name, int32_t sector1, int32_t sector2, uint8_t mergeMode, float weight, float frac) const { From 778d176babc6333e2dcbda5e746ad448d431e1d6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 6 Oct 2025 22:09:55 +0200 Subject: [PATCH 08/54] GPU TPC: Split TPC Merger Fit() into multiple functions --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 4 +- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 652 +++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 46 +- 4 files changed, 377 insertions(+), 330 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index d9b139b73ab7e..ef5abc7e49a31 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -571,7 +571,7 @@ GPUd() int32_t GPUTPCGMMerger::RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack trk.QPt() = inTrack->Param().GetQPt(); trk.TOffset() = Param().par.continuousTracking ? GetConstantMem()->calibObjects.fastTransform->convZOffsetToVertexTime(sector, inTrack->Param().GetZOffset(), Param().continuousMaxTimeBin) : 0; const auto tmp = sectorTrack.ClusterTN() > sectorTrack.ClusterT0() ? std::array{sectorTrack.ClusterTN(), sectorTrack.ClusterT0()} : std::array{sectorTrack.ClusterT0(), sectorTrack.ClusterTN()}; - trk.ShiftZ(this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead + trk.ShiftZ(*this, sector, tmp[0], tmp[1], inTrack->Param().GetX()); // We do not store the inner / outer cluster X, so we just use the track X instead sectorTrack.SetX2(0.f); for (int32_t way = 0; way < 2; way++) { if (way) { @@ -1858,7 +1858,7 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in const auto& GPUrestrict() cls = GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[cl0.num].getTime(), zn = cls[cln.num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(cln.row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(cl0.row)}; - trk.Param().ShiftZ(this, cl0.sector, tmp[0], tmp[1], tmp[2]); + trk.Param().ShiftZ(*this, cl0.sector, tmp[0], tmp[1], tmp[2]); updTrk = &trk; while (updTrk->PrevSegment() >= 0) { auto next = &mMergedTracks[updTrk->PrevSegment()]; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index d3ea60b4dc099..083cdf76699a0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -23,7 +23,7 @@ GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThread { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, &merger, rebuilt); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); }); } @@ -31,7 +31,7 @@ template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), uint32_t, i, merger.Memory()->nLoopData, { - GPUTPCGMTrackParam::PropagateLooper(&merger, i); + GPUTPCGMTrackParam::PropagateLooper(merger, i); }); } @@ -239,7 +239,6 @@ GPUdii() void GPUTPCGMMergerHitWeights::Thread GPUdii() void GPUTPCGMMergerHitWeights::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t iteration) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 42b65abeca828..fdfddec220604 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,13 +47,13 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_t iTrk, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { - constexpr float kDeg2Rad = M_PI / 180.f; constexpr float maxSinPhi = constants::MAX_SIN_PHI; CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); - const GPUParam& GPUrestrict() param = merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); + GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); GPUdEdx dEdx, dEdxAlt; GPUTPCGMPropagator prop; @@ -86,13 +86,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); ResetCovariance(); prop.SetSeedingErrors(!(refit)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); - prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); @@ -128,13 +127,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger->GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger->GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); + // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) + // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; @@ -169,253 +168,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } } - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - const GPUTPCTracker& GPUrestrict() tracker = *(merger -> GetConstantMem()->tpcTrackers + cluster.sector); - const GPUTPCRow& GPUrestrict() row = tracker.Row(cluster.row); - GPUglobalref() const cahit2* hits = tracker.HitData(row); - GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - if (row.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float zOffset = param.par.continuousTracking ? merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(cluster.sector, mTOffset, param.continuousMaxTimeBin) : 0; - const float y0 = row.Grid().YMin(); - const float stepY = row.HstepY(); - const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway - const float stepZ = row.HstepZ(); - int32_t bin, ny, nz; - - float err2Y, err2Z; - param.GetClusterErrors2(cluster.sector, cluster.row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - - float uncorrectedZ; - merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, ImP0, ImP1, uncorrectedY, uncorrectedZ); - - int32_t nCandidates = 0; - if (CAMath::Abs(uncorrectedY) <= row.getTPCMaxY()) { - const float kFactor = tracker.GetChiSeedFactor(); - const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? - const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); - const float tubeY = CAMath::Sqrt(sy2); - const float tubeZ = CAMath::Sqrt(sz2); - row.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); - - const int32_t nBinsY = row.Grid().Ny(); - const int32_t idOffset = tracker.Data().ClusterIdOffset(); - const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - for (int32_t k = 0; k <= nz; k++) { - const int32_t mybin = bin + k * nBinsY; - const uint32_t hitFst = firsthit[mybin]; - const uint32_t hitLst = firsthit[mybin + ny + 1]; - for (uint32_t ih = hitFst; ih < hitLst; ih++) { - const cahit2 hh = hits[ih]; - const float y = y0 + hh.x * stepY; - const float z = z0 + hh.y * stepZ; - const float dy = y - uncorrectedY; - const float dz = z - uncorrectedZ; - - if (dy * dy < sy2 && dz * dz < sz2) { - float err2YA, err2ZA; - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; - const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; - const float time = cl.getTime(); - const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); - const float invCharge = 1.f / cl.qMax; - float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); - invAvgCharge *= invAvgCharge; - - prop.GetErr2(err2YA, err2ZA, merger->Param(), mP[1], cluster.row, clflags, cluster.sector, time, invAvgCharge, invCharge); - const float Jw0 = 1.f / (ImC0 + err2YA); - const float Jw2 = 1.f / (ImC2 + err2ZA); - const float chi2Y = Jw0 * dy * dy; - const float chi2Z = Jw2 * dz * dz; - bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); - float err = dy * dy + dz * dz; - if (ok) { - int32_t insert = nCandidates; - for (int32_t c = 0; c < nCandidates; c++) { - if (err < merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { - insert = c; - break; - } - } - if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { - for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = cluster.row, .sector = cluster.sector, .error = err, .weight = 0, .best = 0}; - nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); - } - } - } - } - } - CADEBUG(const auto* dbgCand = &merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Hit %d Candidate %d hit %d err %f\n", iTrk, (int)cluster.row, cluster.num, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); - } - if (nCandidates == 0) { - merger->ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; - } - } - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); + if (retValHit == 1) { break; - } - if (retValProp || sinPhiErr) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - NTolerated++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + } else if (retValHit == -1) { continue; } - CADEBUG(printf("\n")); - - int32_t retValUpd = 0, retValInt = 0; - float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); - if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; - } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - - float err2Y, err2Z; - const float time = merger->GetConstantMem()->ioPtrs.clustersNative ? merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; - const float invSqrtCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - const float invCharge = merger->GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; - float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; - invAvgCharge *= invAvgCharge; - prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); - } - } - - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters - retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; - } else { - retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); - } - GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { - merger->DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); - }); - } - // clang-format off - CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); - CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); - // clang-format on - - ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! - if (!retValUpd && !retValInt) // track is updated - { - lastUpdateX = mX; - covYYUpd = mC[0]; - nMissed = nMissed2 = 0; - UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters - bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; - if (acc || accAlt) { - float qtot = 0, qmax = 0, pad = 0, relTime = 0; - const int32_t clusterCount = (ihit - ihitMergeFirst) * wayDirection + 1; - for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { - const ClusterNative& cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num]; - qtot += cl.getQtot(); - qmax = CAMath::Max(qmax, cl.qMax); - pad += cl.getPad(); - relTime += cl.getTime(); - } - qtot /= clusterCount; // TODO: Weighted Average - pad /= clusterCount; - relTime /= clusterCount; - relTime = relTime - CAMath::Round(relTime); - if (acc) { - dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - if (accAlt) { - dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], merger->GetConstantMem()->calibObjects, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); - } - } - } - } - } - } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track - if (retValInt || allowChangeClusters) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); - } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - } - if (!retValInt) { - nMissed++; - nMissed2++; - } - } else { - break; // bad chi2 for the whole track, stop the fit - } } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); @@ -449,9 +209,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ } if (param.par.dodEdx && param.dodEdxEnabled) { - dEdx.computedEdx(merger->MergedTracksdEdx()[iTrk], param); + dEdx.computedEdx(merger.MergedTracksdEdx()[iTrk], param); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.computedEdx(merger->MergedTracksdEdxAlt()[iTrk], param); + dEdxAlt.computedEdx(merger.MergedTracksdEdxAlt()[iTrk], param); } } Alpha = prop.GetAlpha(); @@ -461,6 +221,277 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger* GPUrestrict() merger, int32_ return true; } +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const int32_t nWays = param.rec.tpc.nWays; + const int32_t wayDirection = (iWay & 1) ? -1 : 1; + const auto& cluster = clusters[ihit]; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + // clang-format on + if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + { + CADEBUG(printf("REROTATE\n")); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { + retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + } + } + if (retValProp == 0) { + lastRow = cluster.row; + lastSector = cluster.sector; + } + // clang-format off + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } + + static constexpr float kDeg2Rad = M_PI / 180.f; + const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + return 1; + } + if (retValProp || sinPhiErr) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + NTolerated++; + CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + return -1; + } + CADEBUG(printf("\n")); + + int32_t retValUpd = 0, retValInt = 0; + float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); + if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + } else { + int8_t rejectChi2 = 0; + if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { + if (iWay == nWays - 3) { + rejectChi2 = GPUTPCGMPropagator::rejectInterFill; + } else if (iWay == nWays - 2) { + rejectChi2 = GPUTPCGMPropagator::rejectInterReject; + } else if (iWay == nWays - 1) { + rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; + } + } + + float err2Y, err2Z; + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); + + if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { + if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = GPUTPCGMPropagator::rejectDirect; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + } + } + + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; + } else { + retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); + } + GPUCA_DEBUG_STREAMER_CHECK(if (o2::utils::DebugStreamer::checkStream(o2::utils::StreamFlags::streamUpdateTrack, iTrk)) { + merger.DebugStreamerUpdate(iTrk, ihit, xx, yy, zz, cluster, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num], *this, prop, inter, rejectChi2, refit, retValUpd, sumInvSqrtCharge / nAvgCharge * sumInvSqrtCharge / nAvgCharge, yy, zz, clusterState, retValInt, err2Y, err2Z); + }); + } + // clang-format off + CADEBUG(if (!CheckCov()) GPUError("INVALID COV AFTER UPDATE!!!")); + CADEBUG(printf("\t%21sFit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd, retValInt)); + // clang-format on + + ConstrainSinPhi(); // TODO: Limit using ConstrainSinPhi everywhere! + if (!retValUpd && !retValInt) // track is updated + { + lastUpdateX = mX; + covYYUpd = mC[0]; + nMissed = nMissed2 = 0; + UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + N++; + ihitStart = ihit; + float dy = mP[0] - prop.Model().Y(); + float dz = mP[1] - prop.Model().Z(); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } + FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track + if (retValInt || allowChangeClusters) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); + } else if (finalFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + if (!retValInt) { + nMissed++; + nMissed2++; + } + } else { + return 1; // bad chi2 for the whole track, stop the fit + } + return 0; +} + +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); + const GPUTPCRow& GPUrestrict() rowData = tracker.Row(row); + GPUglobalref() const cahit2* hits = tracker.HitData(rowData); + GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); + float uncorrectedY = -1e6f, uncorrectedZ; + if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; + const float y0 = rowData.Grid().YMin(); + const float stepY = rowData.HstepY(); + const float z0 = rowData.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway + const float stepZ = rowData.HstepZ(); + int32_t bin, ny, nz; + + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + + int32_t nCandidates = 0; + if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + const float kFactor = tracker.GetChiSeedFactor(); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float tubeY = CAMath::Sqrt(sy2); + const float tubeZ = CAMath::Sqrt(sz2); + rowData.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + + const int32_t nBinsY = rowData.Grid().Ny(); + const int32_t idOffset = tracker.Data().ClusterIdOffset(); + const int32_t* ids = &(tracker.Data().ClusterDataIndex()[rowData.HitNumberOffset()]); + for (int32_t k = 0; k <= nz; k++) { + const int32_t mybin = bin + k * nBinsY; + const uint32_t hitFst = firsthit[mybin]; + const uint32_t hitLst = firsthit[mybin + ny + 1]; + for (uint32_t ih = hitFst; ih < hitLst; ih++) { + const cahit2 hh = hits[ih]; + const float y = y0 + hh.x * stepY; + const float z = z0 + hh.y * stepZ; + const float dy = y - uncorrectedY; + const float dz = z - uncorrectedZ; + + if (dy * dy < sy2 && dz * dz < sz2) { + float err2YA, err2ZA; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[idOffset + ids[ih]]; + const auto clflags = cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + const float time = cl.getTime(); + const float invSqrtCharge = CAMath::InvSqrt(cl.qMax); + const float invCharge = 1.f / cl.qMax; + float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); + invAvgCharge *= invAvgCharge; + + prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + const float Jw0 = 1.f / (ImC0 + err2YA); + const float Jw2 = 1.f / (ImC2 + err2ZA); + const float chi2Y = Jw0 * dy * dy; + const float chi2Z = Jw2 * dz * dz; + bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); + float err = dy * dy + dz * dz; + if (ok) { + int32_t insert = nCandidates; + for (int32_t c = 0; c < nCandidates; c++) { + if (err < merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + insert = c; + break; + } + } + if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { + for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; + } + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; + nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); + } + } + } + } + } + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + } + if (nCandidates == 0) { + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + } + } + return uncorrectedY; +} + +GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +{ + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters + bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; + if (acc || accAlt) { + float qtot = 0, qmax = 0, pad = 0, relTime = 0; + const int32_t clusterCount = CAMath::Abs(ihit - ihitMergeFirst) + 1; + for (int32_t iTmp = ihitMergeFirst; iTmp != ihit + wayDirection; iTmp += wayDirection) { + const ClusterNative& cl = clustersArray[clusters[iTmp].num]; + qtot += cl.getQtot(); + qmax = CAMath::Max(qmax, cl.qMax); + pad += cl.getPad(); + relTime += cl.getTime(); + } + qtot /= clusterCount; // TODO: Weighted Average + pad /= clusterCount; + relTime /= clusterCount; + relTime = relTime - CAMath::Round(relTime); + const auto& cluster = clusters[ihit]; + if (acc) { + dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + if (accAlt) { + dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); + } + } + } + } + } +} + GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) { constexpr float kDeg2Rad = M_PI / 180.f; @@ -518,11 +549,11 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger* GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger->Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge maxDistY = (maxDistY + mC[0]) * 20.f; maxDistZ = (maxDistZ + mC[2]) * 20.f; int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) @@ -537,10 +568,10 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t xx = yy = zz = 0.f; clusterState = 0; while (true) { - const ClusterNative& GPUrestrict() cl = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; float clamp = cl.getQtot(); float clx, cly, clz; - merger->GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); float dy = cly - projY; float dz = clz - projZ; if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { @@ -572,25 +603,25 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t return 0; } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) { float Y, Z; float X = 0; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, mP[0], mP[1], X); if (prop.GetPropagatedYZ(X, Y, Z)) { Y = mP[0]; Z = mP[1]; } - return AttachClusters(Merger, sector, iRow, iTrack, goodLeg, Y, Z); + return AttachClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) +GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { - const auto& param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); if (param.rec.tpc.disableRefitAttachment & 1) { return -1e6f; } - const GPUTPCTracker& GPUrestrict() tracker = *(Merger->GetConstantMem()->tpcTrackers + sector); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); @@ -598,7 +629,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return -1e6f; } - const float zOffset = param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... + const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... const float y0 = row.Grid().YMin(); const float stepY = row.HstepY(); const float z0 = row.Grid().ZMin() - zOffset; // We can use our own ZOffset, since this is only used temporarily anyway @@ -606,7 +637,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric int32_t bin, ny, nz; float uncorrectedY, uncorrectedZ; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { return uncorrectedY; } @@ -618,7 +649,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const float tubeMinSize2 = protect ? param.rec.tpc.tubeProtectMinSize2 : 0.f; float tubeSigma2 = protect ? param.rec.tpc.tubeProtectSigma2 : param.rec.tpc.tubeRemoveSigma2; uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); - float time = Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? Merger->GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform + float time = merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformInTimeFrame(sector, uncorrectedZ + (param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0), param.continuousMaxTimeBin); // TODO: Simplify this call in TPCFastTransform if (iRow < param.rec.tpc.tubeExtraProtectMinRow || pad < param.rec.tpc.tubeExtraProtectEdgePads || pad >= (uint32_t)(GPUTPCGeometry::NPads(iRow) - param.rec.tpc.tubeExtraProtectEdgePads) || param.GetUnscaledMult(time) / GPUTPCGeometry::Row2X(iRow) > param.rec.tpc.tubeExtraProtectMinOccupancy) { @@ -635,8 +666,8 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric const int32_t nBinsY = row.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - uint32_t myWeight = Merger->TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; - GPUAtomic(uint32_t)* const weights = Merger->ClusterAttachment(); + uint32_t myWeight = merger.TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; + GPUAtomic(uint32_t)* const weights = merger.ClusterAttachment(); if (goodLeg) { myWeight |= gputpcgmmergertypes::attachGoodLeg; } @@ -670,10 +701,10 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) +GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) { constexpr float kSectAngle = 2 * M_PI / 18.f; - if (Merger->Param().rec.tpc.disableRefitAttachment & 2) { + if (merger.Param().rec.tpc.disableRefitAttachment & 2) { return dodEdx; } if (CAMath::Abs(lastRow - toRow) < 2) { @@ -694,14 +725,14 @@ GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger* GP } if (dodEdx && iRow + step == toRow) { float yUncorrected, zUncorrected; - Merger->GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (Merger->GetConstantMem()->calibObjects.dEdxCalibContainer && Merger->GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { dodEdx = false; } } CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(Merger, sector, iRow, iTrack, goodLeg, prop); + AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); } return dodEdx; } @@ -719,7 +750,7 @@ GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterPar outerParam->alpha = alpha; } -GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) +GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha) { if (iRow == 0 || iRow == GPUTPCGeometry::NROWS - 1) { return; @@ -734,10 +765,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* return; } - uint32_t nLoopData = CAMath::AtomicAdd(&Merger->Memory()->nLoopData, 1u); - if (nLoopData >= Merger->NMaxTracks()) { - Merger->raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, Merger->NMaxTracks()); - CAMath::AtomicExch(&Merger->Memory()->nLoopData, Merger->NMaxTracks()); + uint32_t nLoopData = CAMath::AtomicAdd(&merger.Memory()->nLoopData, 1u); + if (nLoopData >= merger.NMaxTracks()) { + merger.raiseError(GPUErrors::ERROR_MERGER_LOOPER_OVERFLOW, nLoopData, merger.NMaxTracks()); + CAMath::AtomicExch(&merger.Memory()->nLoopData, merger.NMaxTracks()); return; } GPUTPCGMLoopData data; @@ -747,36 +778,36 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger* data.sector = sector; data.row = iRow; data.outwards = outwards; - Merger->LoopData()[nLoopData] = data; + merger.LoopData()[nLoopData] = data; } -GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx) +GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { GPUTPCGMPropagator prop; prop.SetMaterialTPC(); - prop.SetPolynomialField(&Merger->Param().polynomialField); + prop.SetPolynomialField(&merger.Param().polynomialField); prop.SetMaxSinPhi(constants::MAX_SIN_PHI); - prop.SetMatLUT(Merger->Param().rec.useMatLUT ? Merger->GetConstantMem()->calibObjects.matLUT : nullptr); + prop.SetMatLUT(merger.Param().rec.useMatLUT ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetSeedingErrors(false); prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = Merger->LoopData()[loopIdx]; + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (false) { - data.param.AttachClustersLooper(Merger, data.sector, data.row, data.track, data.outwards, prop); + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } else { - data.param.AttachClustersLooperFollow(Merger, prop, data.sector, data.track, data.outwards); + data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) { float toX = mX; - bool inFlyDirection = (Merger->MergedTracks()[iTrack].Leg() & 1) ^ up; + bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; constexpr float kSectAngle = 2 * M_PI / 18.f; - const GPUParam& GPUrestrict() param = Merger->Param(); + const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; float lrFactor = right ^ !up ? 1.f : -1.f; @@ -813,7 +844,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float rowX = GPUTPCGeometry::Row2X(j); if (CAMath::Abs(rowX - (-mP[0] * lrFactor)) < 1.5f) { CADEBUG(printf("\t\tAttempt row %d (X %f Y %f Z %f)\n", j, rowX, mX * lrFactor, mP[1])); - AttachClusters(Merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); + AttachClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); } } } @@ -839,7 +870,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) { constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -889,31 +920,32 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger* GPUr float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; if (CAMath::Abs(rowX - paramX) < 1.5f) { // printf("Attempt row %d at y %f\n", j, X); - AttachClusters(Merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); + AttachClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); } } } } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, int32_t N) { if (N == 0) { N = 1; } - const auto& GPUrestrict() cls = merger->GetConstantMem()->ioPtrs.clustersNative->clustersLinear; + const auto& GPUrestrict() cls = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; float z0 = cls[clusters[0].num].getTime(), zn = cls[clusters[N - 1].num].getTime(); const auto tmp = zn > z0 ? std::array{zn, z0, GPUTPCGeometry::Row2X(clusters[N - 1].row)} : std::array{z0, zn, GPUTPCGeometry::Row2X(clusters[0].row)}; return ShiftZ(merger, clusters[0].sector, tmp[0], tmp[1], tmp[2]); } -GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float clx) +GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger& GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float clx) { - if (!merger->Param().par.continuousTracking) { + const GPUParam& GPUrestrict() param = merger.Param(); + if (!param.par.continuousTracking) { return 0.f; } float deltaZ = 0.f; bool beamlineReached = false; - const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * merger->Param().polynomialField.GetNominalBz())); + const float r1 = CAMath::Max(0.0001f, CAMath::Abs(mP[4] * param.polynomialField.GetNominalBz())); if (r1 < 0.01501) { // 100 MeV @ 0.5T ~ 0.66m cutof const float dist2 = mX * mX + mP[0] * mP[0]; const float dist1r2 = dist2 * r1 * r1; @@ -940,16 +972,16 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg } if (!beamlineReached) { - float refZ = ((sector < GPUTPCGeometry::NSECTORS / 2) ? merger->Param().rec.tpc.defaultZOffsetOverR : -merger->Param().rec.tpc.defaultZOffsetOverR) * clx; + float refZ = ((sector < GPUTPCGeometry::NSECTORS / 2) ? param.rec.tpc.defaultZOffsetOverR : -param.rec.tpc.defaultZOffsetOverR) * clx; float basez; - merger->GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); deltaZ = basez - refZ; } { - float deltaT = merger->GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + float deltaT = merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); mTOffset += deltaT; - const float maxT = cltmin - merger->GetConstantMem()->calibObjects.fastTransform->getT0(); - const float minT = cltmax - merger->GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); + const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); + const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); deltaT = 0.f; if (mTOffset < minT) { @@ -959,7 +991,7 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger* GPUrestrict() merg deltaT = maxT - mTOffset; } if (deltaT != 0.f) { - deltaZ += merger->GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); + deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); mTOffset += deltaT; } @@ -1001,7 +1033,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger* GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. { if (!track.OK()) { return; @@ -1015,27 +1047,27 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, merger->Clusters() + track.FirstClusterRef(), nTrackHits, NTolerated, Alpha, track, rebuilt); - CADEBUG(if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } - CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger->MergedTracks()[iTrk].Looper()); }); + CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? - if (t.GetNDF() <= 0 && !rebuilt && merger->Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? + track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { track.Param() = t; track.Alpha() = Alpha; } - if (!merger->Param().rec.tpc.rebuildTrackInFit || rebuilt) { + if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) { track.SetNClustersFitted(nTrackHits); } - // if (track.OK()) merger->DebugRefitMergedTrack(track); + // if (track.OK()) merger.DebugRefitMergedTrack(track); } GPUd() void GPUTPCGMTrackParam::Rotate(float alpha) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index b287e5a717edf..a4ca605e5d547 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -27,8 +27,14 @@ class AliExternalTrackParam; +namespace o2::tpc +{ +struct ClusterNative; +}; + namespace o2::gpu { +class GPUTPCTracker; class GPUTPCGMMerger; class GPUTPCGMBorderTrack; struct GPUParam; @@ -36,6 +42,13 @@ class GPUTPCGMPhysicalTrackModel; class GPUTPCGMPolynomialField; class GPUTPCGMMergedTrack; class GPUTPCGMPropagator; +class GPUdEdx; +struct GPUTPCGMMergedTrackHit; + +namespace gputpcgmmergertypes +{ +struct InterpolationErrorHit; +} // namespace gputpcgmmergertypes /** * @class GPUTPCGMTrackParam @@ -141,23 +154,26 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger* merger, int32_t iTrk, GPUTPCGMMergedTrackHit* clusters, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger* merger, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); - - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger* GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + + GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); - GPUd() static void PropagateLooper(const GPUTPCGMMerger* GPUrestrict() Merger, int32_t loopIdx); + GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); - GPUd() void AddCovDiagErrors(const float* GPUrestrict() errors2); - GPUd() void AddCovDiagErrorsWithCorrelations(const float* GPUrestrict() errors2); + GPUd() void AddCovDiagErrors(const float* errors2); + GPUd() void AddCovDiagErrorsWithCorrelations(const float* errors2); GPUdi() void MarkClusters(GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihitFirst, int32_t ihitLast, int32_t wayDirection, uint8_t state) { @@ -183,8 +199,8 @@ class GPUTPCGMTrackParam } GPUd() void Rotate(float alpha); - GPUd() float ShiftZ(const GPUTPCGMMerger* merger, uint32_t sector, float cltmax, float cltmin, float clx); - GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger* merger, int32_t N); + GPUd() float ShiftZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float clx); + GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, int32_t N); GPUd() static float Reciprocal(float x) { return 1.f / x; } GPUdi() static void Assign(float& x, bool mask, float v) From de294685184064f6ae1f055c22d5e4dd04d7c0ad Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:01 +0200 Subject: [PATCH 09/54] GPU: Add looperFollowMode option --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index f14a757d63970..1faec37482aa0 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -144,6 +144,7 @@ AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fractio AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index fdfddec220604..29072163ec067 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -794,10 +794,10 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); - if (false) { - data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); - } else { + if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); + } else { + data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); } } From d3d611030a415fa81dda96a909b11f0fe71f6f0b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 23 Oct 2025 13:25:20 +0200 Subject: [PATCH 10/54] GPU: simplifaction, refactoring and some minor improvements --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 145 ++++++------- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 196 ++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 13 +- 5 files changed, 174 insertions(+), 194 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 1faec37482aa0..cac6faf8e0908 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -138,7 +138,7 @@ AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger ( AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") -AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits, <0: reject at max -n hits") +AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 974a0b3ea2074..49e0773abab1d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -608,7 +608,7 @@ GPUd() float GPUTPCGMPropagator::PredictChi2(float posY, float posZ, float err2Y } } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge) { float err2Y, err2Z; GetErr2(err2Y, err2Z, param, posZ, iRow, clusterState, sector, time, avgInvCharge, invCharge); @@ -616,7 +616,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, iRow, param, clusterState, rejectChi2, refit, err2Y, err2Z); } -GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z) +GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, const GPUParam& GPUrestrict() param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z) { if (mT->NDF() == -5) { // first measurement: no need to filter, as the result is known in advance. just set it. mT->ResetCovariance(); @@ -635,80 +635,83 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return 0; } - return Update(posY, posZ, clusterState, rejectChi2 == rejectDirect, err2Y, err2Z, ¶m); + return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); - if (rejectChi2 == rejectInterFill) { - inter->posY = mP[0]; - inter->posZ = mP[1]; - if (mT->NDF() <= 0) { - inter->errorY = inter->errorZ = 100.f; - } else { - inter->errorY = mC[0]; - inter->errorZ = mC[2]; - } - } else if (rejectChi2 == rejectInterReject) { - float chi2Y, chi2Z; - if (mT->NDF() <= 0) { - chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); - chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); - } else if (mFitInProjections) { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - const float Jw0 = 1.f / (ImC0 + err2Y); - const float Jw2 = 1.f / (ImC2 + err2Z); - chi2Y = Jw0 * Jz0 * Jz0; - chi2Z = Jw2 * Jz1 * Jz1; - } else { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - float Iw0 = mC[2] + (float)inter->errorZ; - float Iw2 = mC[0] + (float)inter->errorY; - float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); - Idet = 1.f / Idet; - Iw0 *= Idet; - const float Iw1 = mC[1] * Idet; - Iw2 *= Idet; - const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; - const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; - const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; - const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; - const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; - const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; - const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - float Jw0 = ImC2 + err2Z; - float Jw2 = ImC0 + err2Y; - float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); - Jdet = 1.f / Jdet; - Jw0 *= Jdet; - const float Jw1 = ImC1 * Jdet; - Jw2 *= Jdet; - chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); - chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); - } - if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? - // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); - return updateErrorClusterRejectedInInterpolation; - } + inter->posY = mP[0]; + inter->posZ = mP[1]; + if (mT->NDF() <= 0) { + inter->errorY = inter->errorZ = 100.f; + } else { + inter->errorY = mC[0]; + inter->errorZ = mC[2]; + } +} + +GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) +{ + float* GPUrestrict() mC = mT->Cov(); + float* GPUrestrict() mP = mT->Par(); + float chi2Y, chi2Z; + if (mT->NDF() <= 0) { + chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); + chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); + } else if (mFitInProjections) { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0; + const float ImP1 = mP[1] + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0]; + const float ImC2 = mC[2] - Ik11 * mC[2]; + // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + const float Jw0 = 1.f / (ImC0 + err2Y); + const float Jw2 = 1.f / (ImC2 + err2Z); + chi2Y = Jw0 * Jz0 * Jz0; + chi2Z = Jw2 * Jz1 * Jz1; + } else { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + float Iw0 = mC[2] + (float)inter->errorZ; + float Iw2 = mC[0] + (float)inter->errorY; + float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); + Idet = 1.f / Idet; + Iw0 *= Idet; + const float Iw1 = mC[1] * Idet; + Iw2 *= Idet; + const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; + const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; + const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; + const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; + const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; + const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; + const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; + const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; + const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + float Jw0 = ImC2 + err2Z; + float Jw2 = ImC0 + err2Y; + float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); + Jdet = 1.f / Jdet; + Jw0 *= Jdet; + const float Jw1 = ImC1 * Jdet; + Jw2 *= Jdet; + chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); + chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); + } + if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? + // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); + return updateErrorClusterRejectedInInterpolation; } return 0; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index c3f7aac84a1a5..bf67d027f6b05 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -57,11 +57,6 @@ class GPUTPCGMPropagator updateErrorClusterRejectedInUpdate = 4, updateErrorClusterRejectedEdge = 5 }; - enum RejectChi2Mode { - rejectDirect = 1, - rejectInterFill = 2, - rejectInterReject = 3 - }; GPUdDefault() GPUTPCGMPropagator() = default; @@ -104,10 +99,11 @@ class GPUTPCGMPropagator GPUd() int32_t PropagateToXAlphaBz(float posX, float posAlpha, bool inFlyDirection); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); - GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, int8_t rejectChi2, bool refit, float err2Y, float err2Z); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); + GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, int8_t rejectChi2, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); + GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 29072163ec067..1ee9cc8115954 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,7 +50,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; - CADEBUG(static constexpr float kSectAngle = 2 * M_PI / 18.f); + constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -104,11 +104,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t wayDirection = (iWay & 1) ? -1 : 1; for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); - if (crossCE) { - lastSector = clusters[ihit].sector; - } - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { @@ -130,11 +125,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); } - // clang-format off CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - // CADEBUG(if ((uint32_t)merger.GetTrackingChain()->mIOPtrs.nMCLabelsTPC > clusters[ihit].num)) - // CADEBUG({printf(" MC:"); for (int32_t i = 0; i < 3; i++) {int32_t mcId = merger.GetTrackingChain()->mIOPtrs.mcLabelsTPC[clusters[ihit].num].fClusterID[i].fMCID; if (mcId >= 0) printf(" %d", mcId); } } printf("\n")); - // clang-format on if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { nMissed++; nMissed2++; @@ -151,33 +142,64 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const auto& cluster = clusters[ihit]; - - // clang-format off - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - // clang-format on - if (!param.rec.tpc.rebuildTrackInFit && allowChangeClusters && lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; - dodEdx = AttachClustersPropagate(merger, cluster.sector, lastRow, cluster.row, iTrk, track.Leg() == 0, prop, inFlyDirection, constants::MAX_SIN_PHI, dodEdx); - if (dodEdx) { - dEdx.fillSubThreshold(lastRow - wayDirection); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(lastRow - wayDirection); + if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (dodEdx || doAttach || doIntertpolate) { + int32_t step = cluster.row > lastRow ? 1 : -1; + float tmpX, tmpY, tmpZ; + for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { + if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { + break; + } + if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + break; + } + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { + dEdx.fillSubThreshold(iRow); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { + dEdxAlt.fillSubThreshold(iRow); + } + } + } + } + if (doAttach) { + AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); } } } } - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - int32_t retValHit = FitHit(merger, iTrk, track, N, NTolerated, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, lastRow, lastSector, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, ihitStart, allowChangeClusters, refit, finalFit, nMissed, nMissed2, covYYUpd, resetT0); - if (retValHit == 1) { + auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + if (retValHit == 0) { + ihitStart = ihit; + N++; + covYYUpd = mC[0]; + } else if (retValHit == 1) { break; - } else if (retValHit == -1) { + } else if (retValHit == 2) { + NTolerated++; continue; } + + lastRow = cluster.row; + lastSector = cluster.sector; } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255 && lastSector != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -221,7 +243,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -238,26 +260,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); } } - if (retValProp == 0) { - lastRow = cluster.row; - lastSector = cluster.sector; - } // clang-format off CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); @@ -269,33 +275,33 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; + return 1; // bad chi2 for the whole track, stop the fit } if (retValProp || sinPhiErr) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - NTolerated++; CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); - return -1; + return 2; // Propagate failed or high incl angle } CADEBUG(printf("\n")); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } + int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; } else { - int8_t rejectChi2 = 0; - if (param.rec.tpc.mergerInterpolateErrors && CAMath::Abs(ihit - ihitMergeFirst) <= 1) { - if (iWay == nWays - 3) { - rejectChi2 = GPUTPCGMPropagator::rejectInterFill; - } else if (iWay == nWays - 2) { - rejectChi2 = GPUTPCGMPropagator::rejectInterReject; - } else if (iWay == nWays - 1) { - rejectChi2 = (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) ? GPUTPCGMPropagator::rejectDirect : 0; - } - } - float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].qMax) : 0.f; @@ -305,12 +311,23 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - if (rejectChi2 >= GPUTPCGMPropagator::rejectInterFill) { - if (rejectChi2 == GPUTPCGMPropagator::rejectInterReject && inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = GPUTPCGMPropagator::rejectDirect; - } else { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, rejectChi2, &inter, err2Y, err2Z, deltaZ); + bool rejectChi2 = false; + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 3) { + prop.InterpolateFill(yy, zz, &inter); + } else if (iWay == nWays - 2) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else if (!param.rec.tpc.rebuildTrackInFit) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { + rejectChi2 = true; + } } + } else { + rejectChi2 = GetNDF() > 0 && allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters @@ -331,11 +348,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (!retValUpd && !retValInt) // track is updated { lastUpdateX = mX; - covYYUpd = mC[0]; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - N++; - ihitStart = ihit; float dy = mP[0] - prop.Model().Y(); float dz = mP[1] - prop.Model().Z(); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { @@ -343,6 +357,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.SetTrack(this, prop.GetAlpha()); } FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -353,10 +368,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger nMissed++; nMissed2++; } + return -1; // cluster rejected } else { return 1; // bad chi2 for the whole track, stop the fit } - return 0; } GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) @@ -392,6 +407,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); int32_t nCandidates = 0; + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + nCandidates++; + } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? @@ -701,42 +719,6 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric return uncorrectedY; } -GPUd() bool GPUTPCGMTrackParam::AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& GPUrestrict() prop, bool inFlyDirection, float maxSinPhi, bool dodEdx) -{ - constexpr float kSectAngle = 2 * M_PI / 18.f; - if (merger.Param().rec.tpc.disableRefitAttachment & 2) { - return dodEdx; - } - if (CAMath::Abs(lastRow - toRow) < 2) { - return dodEdx; - } - int32_t step = toRow > lastRow ? 1 : -1; - float xx = mX - GPUTPCGeometry::Row2X(lastRow); - for (int32_t iRow = lastRow + step; iRow != toRow; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi) { - return dodEdx; - } - if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - return dodEdx; - } - int32_t err = prop.PropagateToXAlpha(xx + GPUTPCGeometry::Row2X(iRow), prop.GetAlpha(), inFlyDirection); - if (err) { - return dodEdx; - } - if (dodEdx && iRow + step == toRow) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); - if (pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { - dodEdx = false; - } - } - CADEBUG(printf("Attaching in row %d\n", iRow)); - AttachClusters(merger, sector, iRow, iTrack, goodLeg, prop); - } - return dodEdx; -} - GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha) { CADEBUG(printf("\t%21sStorO Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f, SP %5.2f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f\n", "", alpha, mX, mP[0], mP[1], mP[4], mP[2], sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]))); @@ -801,7 +783,7 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, int32_t iTrack, bool up) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; @@ -870,7 +852,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! @@ -880,7 +862,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr float SinPhi = CAMath::Sqrt(1 - mP[2] * mP[2]) * (mP[2] > 0 ? -1 : 1); float b = prop.GetBz(prop.GetAlpha(), mX, mP[0], mP[1]); - float dx = outwards ? 1.f : -1.f; + float dx = up ? 1.f : -1.f; const float myRowX = GPUTPCGeometry::Row2X(iRow); // printf("\nAttachMirror sector %d row %d outwards %d\n", (int)sector, (int)iRow, (int)outwards); // printf("X %f Y %f Z %f SinPhi %f -->\n", mX, mP[0], mP[1], mP[2]); @@ -914,7 +896,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr // printf("count %d: At X %f Y %f Z %f SinPhi %f\n", maxTries, mP[2] > 0 ? -Y : Y, mP[2] > 0 ? X : -X, Z, SinPhi); float paramX = mP[2] > 0 ? -Y : Y; - int32_t step = outwards ? 1 : -1; + int32_t step = up ? 1 : -1; int32_t found = 0; for (int32_t j = iRow; j >= 0 && j < (int32_t)GPUTPCGeometry::NROWS && found < 3; j += step) { float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index a4ca605e5d547..d738f09a91c15 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,19 +156,18 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, int32_t& N, int32_t& NTolerated, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, uint8_t& lastRow, uint8_t& lastSector, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, int32_t& ihitStart, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float& covYYUpd, int32_t& resetT0); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); - GPUd() bool AttachClustersPropagate(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t lastRow, int32_t toRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop, bool inFlyDirection, float maxSinPhi = constants::MAX_SIN_PHI, bool checkdEdx = false); - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() Merger, GPUTPCGMPropagator& prop, int32_t sector, int32_t iTrack, bool outwards); - GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() Merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); + GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); + GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From 7e007de5d9deb8de6f53011c2ff81b9fd90f7617 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 13:34:27 +0100 Subject: [PATCH 11/54] GPU TPC: Refactor cluster merging --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 98 +++++++++++-------- 1 file changed, 58 insertions(+), 40 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 1ee9cc8115954..86b179522bfcc 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -121,26 +121,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); - } CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, !param.rec.tpc.rebuildTrackInFit && allowChangeClusters) == -1) { + if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { nMissed++; nMissed2++; continue; } - if (param.rec.tpc.rejectIFCLowRadiusCluster) { - const float r2 = xx * xx + yy * yy; - const float rmax = (83.5f + param.rec.tpc.sysClusErrorMinDist); - if (r2 < rmax * rmax) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); - continue; - } - } - const auto& cluster = clusters[ihit]; if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; @@ -569,55 +556,86 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) { + const int32_t ihitFirst = ihit; + { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + } if (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector) { - float maxDistY, maxDistZ; - prop.GetErr2(maxDistY, maxDistZ, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge - maxDistY = (maxDistY + mC[0]) * 20.f; - maxDistZ = (maxDistZ + mC[2]) * 20.f; - int32_t noReject = 0; // Cannot reject if simple estimation of y/z fails (extremely unlike case) + float maxDistY2, maxDistZ2; + bool noReject = false; // Cannot reject if simple estimation of y/z fails (extremely unlike case) if (CAMath::Abs(clAlpha - prop.GetAlpha()) > 1.e-4f) { noReject = prop.RotateToAlpha(clAlpha); } float projY = 0, projZ = 0; - if (noReject == 0) { + if (!noReject) { noReject |= prop.GetPropagatedYZ(xx, projY, projZ); } - float count = 0.f; - xx = yy = zz = 0.f; - clusterState = 0; - while (true) { - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; - float clamp = cl.getQtot(); - float clx, cly, clz; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); - float dy = cly - projY; - float dz = clz - projZ; - if (noReject == 0 && (dy * dy > maxDistY || dz * dz > maxDistZ)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ), mP[0], projY, cly, mP[1], projZ, clz)); + prop.GetErr2(maxDistY2, maxDistZ2, merger.Param(), zz, clusters[ihit].row, 0, clusters[ihit].sector, -1.f, 0.f, 0.f); // TODO: Use correct time, avgCharge + const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; + maxDistY2 = (maxDistY2 + mC[0]) * kFactor; + maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; + auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + float dy = y - projY; + float dz = z - projZ; + if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { + CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (rejectChi2) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; + clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } + return false; } else { - CADEBUG(printf("\t\tMerging hit row %d X %f Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ihit].row, clx, cly, clz, dy, dz, sqrtf(maxDistY), sqrtf(maxDistZ))); - xx += clx * clamp; // TODO: Weight in pad/time instead of XYZ + CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + return true; + } + }; + const float tmpX = xx; + float count; + if (chkFunction(ihit, yy, zz)) { + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.getQtot(); + xx *= clamp; + yy *= clamp; + zz *= clamp; + clusterState = clusters[ihit].state; + count = clamp; + } else { + xx = yy = zz = count = 0.f; + clusterState = 0; + } + do { + ihit += wayDirection; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[ihit].num]; + const float clamp = cl.getQtot(); + float clx, cly, clz; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(clusters[ihit].sector, clusters[ihit].row, cl.getPad(), cl.getTime(), clx, cly, clz, mTOffset); + if (chkFunction(ihit, cly, clz)) { + xx += clx * clamp; yy += cly * clamp; zz += clz * clamp; clusterState |= clusters[ihit].state; count += clamp; } - if (!(ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector)) { - break; - } - ihit += wayDirection; - } + } while (ihit + wayDirection >= 0 && ihit + wayDirection < maxN && clusters[ihit].row == clusters[ihit + wayDirection].row && clusters[ihit].sector == clusters[ihit + wayDirection].sector); if (count < 0.1f) { CADEBUG(printf("\t\tNo matching cluster in double-row, skipping\n")); + xx = tmpX; return -1; } xx /= count; yy /= count; zz /= count; } + if (merger.Param().rec.tpc.rejectIFCLowRadiusCluster) { + const float r2 = xx * xx + yy * yy; + const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); + if (r2 < rmax2) { + if (rejectChi2) { + MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + return -1; + } + } return 0; } From 5cc435b3019694cef2a9d5d2274777cf92b4f357 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 24 Oct 2025 11:01:17 +0200 Subject: [PATCH 12/54] GPU: Change indexing of interpolated hit temporary data --- GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 5 + GPU/GPUTracking/Global/GPUErrorCodes.h | 35 ++-- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 3 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 173 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 5 +- GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 8 +- 6 files changed, 130 insertions(+), 99 deletions(-) diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index ed334aef3a75f..32fbe83357d6b 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -17,6 +17,9 @@ #include "GPUCommonDef.h" #include "DataFormatsTPC/Constants.h" +#ifndef GPUCA_GPUCODE_DEVICE +#include +#endif namespace o2::gpu { @@ -174,6 +177,8 @@ class GPUTPCGeometry const float v = (sector >= NSECTORS / 2) ? -z : z; return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } + + GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Global/GPUErrorCodes.h b/GPU/GPUTracking/Global/GPUErrorCodes.h index 2080548c7775c..b3894cf267c17 100644 --- a/GPU/GPUTracking/Global/GPUErrorCodes.h +++ b/GPU/GPUTracking/Global/GPUErrorCodes.h @@ -35,22 +35,23 @@ GPUCA_ERROR_CODE(15, ERROR_SECTORDATA_Z_OVERFLOW, Sector, Value) GPUCA_ERROR_CODE(16, ERROR_MERGER_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(17, ERROR_MERGER_REBUILD_HIT_OVERFLOW, Value, Max) GPUCA_ERROR_CODE(18, ERROR_MERGER_TRACK_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(19, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(20, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) -GPUCA_ERROR_CODE(21, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(22, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) -GPUCA_ERROR_CODE(23, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(24, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) -GPUCA_ERROR_CODE(25, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER -GPUCA_ERROR_CODE(26, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped -GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded -GPUCA_ERROR_CODE(28, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete -GPUCA_ERROR_CODE(29, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset -GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word -GPUCA_ERROR_CODE(31, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode -GPUCA_ERROR_CODE(32, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version -GPUCA_ERROR_CODE(33, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder -GPUCA_ERROR_CODE(33, MAX_GPUCA_ERROR_NUMBER) +GPUCA_ERROR_CODE(19, ERROR_MERGER_INTERPOLATION_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(20, ERROR_COMPRESSION_ROW_HIT_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(21, ERROR_LOOPER_MATCH_OVERFLOW, Value, Max) +GPUCA_ERROR_CODE(22, ERROR_CF_PEAK_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(23, ERROR_CF_CLUSTER_OVERFLOW, Sector, Value, Max) +GPUCA_ERROR_CODE(24, ERROR_CF_ROW_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(25, ERROR_CF_GLOBAL_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, ERROR_DECOMPRESSION_ATTACHED_CLUSTER_OVERFLOW, SectorRow, Value, Max) +GPUCA_ERROR_CODE(26, MAX_OVERFLOW_ERROR_NUMBER) // Overflow errors are detected as errno <= MAX_OVERFLOW_ERROR_NUMBER +GPUCA_ERROR_CODE(27, ERROR_TPCZS_INVALID_ROW, SectorRow) // Data from invalid row is skipped +GPUCA_ERROR_CODE(28, ERROR_TPCZS_INVALID_NADC, SectorCRU, SamplesInPage, SamplesWritten) // Invalid number of ADC samples in header, existing samples were decoded +GPUCA_ERROR_CODE(29, ERROR_TPCZS_INCOMPLETE_HBF, SectorCRU, PacketCount, NextPacketCount) // Part of HBF is missing, decoding incomplete +GPUCA_ERROR_CODE(30, ERROR_TPCZS_INVALID_OFFSET, SectorEndpoint, Value, Expected) // Raw page is skipped since it contains invalid payload offset +GPUCA_ERROR_CODE(31, ERROR_TPCZS_INVALID_MAGIC_WORD, Value) // ZS header contains wrong magic word +GPUCA_ERROR_CODE(32, ERROR_TPCZS_PAGE_OVERFLOW, Position, PageEnd) // Ran out of page to decode +GPUCA_ERROR_CODE(33, ERROR_TPCZS_VERSION_MISMATCH, Value, Expected) // ZS decoder received page with wrong version +GPUCA_ERROR_CODE(34, ERROR_TPCZS_UNKNOWN, ErrorCode) // Unkown or invalid error code raised in decoder +GPUCA_ERROR_CODE(34, MAX_GPUCA_ERROR_NUMBER) // #define GPUCA_CHECK_TPCZS_CORRUPTION diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 89b553aba9d1c..d8b14ecc61182 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -38,7 +38,8 @@ struct InterpolationErrorHit { }; struct InterpolationErrors { - InterpolationErrorHit hit[constants::MERGER_MAX_TRACK_CLUSTERS]; + static constexpr size_t size = constants::MERGER_MAX_TRACK_CLUSTERS; + InterpolationErrorHit hit[size]; }; struct GPUResolveSharedMemory : public GPUKernelTemplate::GPUSharedMemoryScan64 { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 86b179522bfcc..612a99de4d2d6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,7 +50,6 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; - constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -62,7 +61,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (int32_t i = 0; i < N; i++) { + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } } @@ -70,6 +69,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t nWays = param.rec.tpc.nWays; const int32_t maxN = N; int32_t ihitStart = 0; + int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; @@ -94,86 +94,81 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); - CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / kSectAngle) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); + CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastRow = 255; - uint8_t lastSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { - CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); - if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { - NTolerated++; - } - if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { + if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { + CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); + if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { + NTolerated++; + } + if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { + clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; + } + continue; } - continue; } const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; + interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); - if (MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters) == -1) { - nMissed++; - nMissed2++; - continue; - } + const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); const auto& cluster = clusters[ihit]; - if (lastRow != 255 && CAMath::Abs(cluster.row - lastRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastRow) == 2; + uint8_t dEdxSubThresholdRow = 255; + bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); + if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - bool doIntertpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); - if (dodEdx || doAttach || doIntertpolate) { - int32_t step = cluster.row > lastRow ? 1 : -1; - float tmpX, tmpY, tmpZ; - for (int32_t iRow = lastRow + step; iRow != cluster.row; iRow += step) { - if (CAMath::Abs(mP[2]) > maxSinPhi || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(kSectAngle / 2.f)) { - break; - } + if (dodEdx || doAttach || doInterpolate) { + int32_t step = cluster.row > lastPropagateRow ? 1 : -1; + for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - break; - } - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { - if (dodEdx) { - float yUncorrected, zUncorrected; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(cluster.sector, iRow, yUncorrected)); - if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(cluster.sector, iRow, pad)))) { - dEdx.fillSubThreshold(iRow); - if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { - dEdxAlt.fillSubThreshold(iRow); - } - } - } - } - if (doAttach) { - AttachClusters(merger, cluster.sector, iRow, iTrk, track.Leg() == 0, prop); - } + FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } + interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; + } + lastPropagateRow = cluster.row; + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + if (doInterpolate) { + FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); + } + continue; } CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - auto& inter = interpolation.hit[iWay & 1 ? ihit : ihitMergeFirst]; - const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (clusters[ihit].sector < 18)); + if ((uint32_t)interpolationIndex >= interpolation.size) { + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); + break; + } + auto& inter = interpolation.hit[interpolationIndex]; + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); if (retValHit == 0) { + DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; + interpolatedStart = interpolationIndex; N++; covYYUpd = mC[0]; } else if (retValHit == 1) { @@ -183,12 +178,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - lastRow = cluster.row; - lastSector = cluster.sector; + lastUpdateRow = cluster.row; + lastUpdateSector = cluster.sector; + assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastRow != 255) { - StoreLoopPropagation(merger, lastSector, lastRow, iTrk, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastRow %d row %d out %d\n", iTrk, (int)lastRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); @@ -230,7 +226,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +{ + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + return; + } + if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + return; + } + const GPUParam& GPUrestrict() param = merger.Param(); + if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + if (dodEdx) { + float yUncorrected, zUncorrected; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, mP[0], mP[1], yUncorrected, zUncorrected); + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, yUncorrected)); + if (!(pad >= GPUTPCGeometry::NPads(iRow) || (merger.GetConstantMem()->calibObjects.dEdxCalibContainer && merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad)))) { + *dEdxSubThresholdRow = iRow; + } + } + } + if (doAttach) { + AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -343,8 +363,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } - FitdEdx(dEdx, dEdxAlt, param, finalFit, merger.GetConstantMem()->calibObjects, ihit, ihitMergeFirst, wayDirection, merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear, clusters, clusterState, zz); - return 0; // ok + return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); @@ -464,9 +483,12 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, const GPUParam& GPUrestrict() param, bool finalFit, const GPUCalibObjectsConst& GPUrestrict() calib, int ihit, int ihitMergeFirst, int wayDirection, const ClusterNative* GPUrestrict() clustersArray, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz) +GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { + const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { + const GPUCalibObjectsConst& GPUrestrict() calib = merger.GetConstantMem()->calibObjects; + const ClusterNative* GPUrestrict() clustersArray = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear; if (param.dodEdxEnabled && finalFit) { // TODO: Costimize flag to remove, and option to remove double-clusters bool acc = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMask) == 0, accAlt = (clusterState & param.rec.tpc.dEdxClusterRejectionFlagMaskAlt) == 0; if (acc || accAlt) { @@ -485,10 +507,16 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& relTime = relTime - CAMath::Round(relTime); const auto& cluster = clusters[ihit]; if (acc) { + if (dEdxSubThresholdRow != 255) { // Must be called before fillCluster, which flushes pending sub-threshold rows with the min charge of their ROC when the ROC changes + dEdx.fillSubThreshold(dEdxSubThresholdRow); + } dEdx.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); } if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMask) != GPUCA_GET_CONSTEXPR(param.rec.tpc, dEdxClusterRejectionFlagMaskAlt)) { if (accAlt) { + if (dEdxSubThresholdRow != 255) { + dEdxAlt.fillSubThreshold(dEdxSubThresholdRow); + } dEdxAlt.fillCluster(qtot, qmax, cluster.row, cluster.sector, mP[2], mP[3], calib, zz, pad, relTime GPUCA_DEBUG_STREAMER_CHECK(, iTrk, clusterState)); } } @@ -497,37 +525,36 @@ GPUdii() void GPUTPCGMTrackParam::FitdEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& } } -GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& Alpha) +GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { constexpr float kDeg2Rad = M_PI / 180.f; - constexpr float kSectAngle = 2 * M_PI / 18.f; if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; } ConstrainSinPhi(); - if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(mP[0]) <= mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } } *this = save; Alpha = saveAlpha; } - if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(kSectAngle / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; } } -GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) +GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, float toY, float toZ, bool inFlyDirection, const GPUParam& GPUrestrict() param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector) { if (mirrorParameters) { prop.Mirror(inFlyDirection); @@ -554,7 +581,7 @@ GPUd() void GPUTPCGMTrackParam::MirrorTo(GPUTPCGMPropagator& GPUrestrict() prop, mChi2 = 0; } -GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, bool rejectChi2) +GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, float& GPUrestrict() xx, float& GPUrestrict() yy, float& GPUrestrict() zz, int32_t maxN, float clAlpha, uint8_t& GPUrestrict() clusterState, const bool markReject) { const int32_t ihitFirst = ihit; { @@ -575,12 +602,12 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, rejectChi2, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); - if (rejectChi2) { + if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; @@ -630,7 +657,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float r2 = xx * xx + yy * yy; const float rmax2 = CAMath::Square(83.5f + merger.Param().rec.tpc.sysClusErrorMinDist); if (r2 < rmax2) { - if (rejectChi2) { + if (markReject) { MarkClusters(clusters, ihitFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); } return -1; @@ -806,7 +833,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; - constexpr float kSectAngle = 2 * M_PI / 18.f; const GPUParam& GPUrestrict() param = merger.Param(); bool right = (mP[2] < 0) ^ up; const int32_t sectorSide = sector >= (int32_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; @@ -821,7 +847,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger CADEBUG(printf("\tRotated: X %f Y %f Z %f SinPhi %f (Alpha %f / %f)\n", mP[0], mX, mP[1], mP[2], prop.GetAlpha(), prop.GetAlpha() + CAMath::Pi() / 2.f)); uint32_t maxTries = 100; while (true) { - while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(kSectAngle / 2.f) + 0.1f) { + while (CAMath::Abs(mX) <= CAMath::Abs(mP[0]) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { if (maxTries-- == 0) { return; } @@ -872,7 +898,6 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { - constexpr float kSectAngle = 2 * M_PI / 18.f; // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; float Y = mP[2] > 0 ? -mX : mX; @@ -907,7 +932,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr Y += dS * SinPhi + h4; Z += dS * mP[3]; SinPhi = newSinPhi; - if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(kSectAngle / 2.f)) { + if (CAMath::Abs(X) > CAMath::Abs(Y) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { // printf("Abort, sector edge\n"); return; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index d738f09a91c15..f0ce761d41418 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,12 +155,13 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); - GPUd() void FitdEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, const GPUParam& param, bool finalFit, const GPUCalibObjectsConst& calib, int ihit, int ihitMergeFirst, int wayDirection, const o2::tpc::ClusterNative* clustersArray, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz); + GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); - GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, bool rejectChi2); + GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index c8d6b00d916cd..7ce2cb82a0a50 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -401,13 +401,11 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { constexpr float kDeg2Rad = M_PI / 180.f; - constexpr float kSectAngle = 2 * M_PI / 18.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { - // a forced energy-loss sign holds along the refit direction: the way to the reference X can run against it - const int32_t eLossSignRef = (mPparam->rec.tpc.trackReferenceX > trk.getX()) == outward ? eLossSign : -eLossSign; + const int32_t eLossSignRef = (mPparam->rec.tpc.trackReferenceX > trk.getX()) == outward ? eLossSign : -eLossSign; // a forced energy-loss sign holds along the refit direction: the way to the reference X can run against it if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef)) { - if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(kSectAngle / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * kSectAngle; + if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef); } From 5318bc302df3c221ef6e156ea376ceb9d399ea40 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 27 Oct 2025 21:35:02 +0100 Subject: [PATCH 13/54] GPU: Move some functionality to CAMath --- GPU/Common/GPUCommonMath.h | 1 + GPU/GPUTracking/DataTypes/GPUTPCGeometry.h | 3 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 9 +++------ GPU/GPUTracking/Refit/GPUTrackingRefit.cxx | 3 +-- GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h | 2 +- GPU/GPUTracking/qa/GPUQA.cxx | 12 ++++++------ GPU/GPUTracking/qa/genEvents.cxx | 4 ++-- GPU/GPUTracking/qa/genEvents.h | 4 ++-- 8 files changed, 18 insertions(+), 20 deletions(-) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index c55b04ddc0743..62efdfb80bb35 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -122,6 +122,7 @@ class GPUCommonMath GPUd() constexpr static double Copysignd(double x, double y); GPUd() constexpr static float TwoPi() { return 6.2831853f; } GPUd() constexpr static float Pi() { return 3.1415927f; } + GPUd() constexpr static float Deg2Rad() { return Pi() / 180.f; } GPUd() constexpr static float Round(float x); GPUd() constexpr static float Floor(float x) { return GPUCA_CHOICE(floorf(x), floorf(x), floor(x)); } GPUd() static uint32_t Float2UIntReint(const float& x); diff --git a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h index 32fbe83357d6b..ac2560a45ef90 100644 --- a/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h +++ b/GPU/GPUTracking/DataTypes/GPUTPCGeometry.h @@ -16,6 +16,7 @@ #define GPUTPCGEOMETRY_H #include "GPUCommonDef.h" +#include "GPUCommonMath.h" #include "DataFormatsTPC/Constants.h" #ifndef GPUCA_GPUCODE_DEVICE #include @@ -178,7 +179,7 @@ class GPUTPCGeometry return (250.f - v) * FACTOR_Z2T; // Used in compression, must remain constant at 250cm } - GPUd() static constexpr float kSectAngle() { return 2 * M_PI / 18.f; } + GPUd() static constexpr float kSectAngle() { return 2 * CAMath::Pi() / 18.f; } }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 612a99de4d2d6..ee70dd488a44e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -278,8 +278,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } - static constexpr float kDeg2Rad = M_PI / 180.f; - const float maxSinForUpdate = CAMath::Sin(70.f * kDeg2Rad); + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { return 1; // bad chi2 for the whole track, stop the fit @@ -527,13 +526,11 @@ GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& G GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestrict() prop, const GPUParam& GPUrestrict() param, float& GPUrestrict() Alpha) { - constexpr float kDeg2Rad = M_PI / 180.f; - if (param.rec.tpc.trackReferenceX <= 500) { GPUTPCGMTrackParam save = *this; float saveAlpha = Alpha; for (int32_t attempt = 0; attempt < 3; attempt++) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Alpha += dAngle; if (prop.PropagateToXAlpha(param.rec.tpc.trackReferenceX, Alpha, 0)) { break; @@ -547,7 +544,7 @@ GPUdni() void GPUTPCGMTrackParam::MoveToReference(GPUTPCGMPropagator& GPUrestric Alpha = saveAlpha; } if (CAMath::Abs(mP[0]) > mX * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float dAngle = CAMath::Round(CAMath::ATan2(mP[0], mX) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); Rotate(dAngle); ConstrainSinPhi(); Alpha += dAngle; diff --git a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx index 7ce2cb82a0a50..906e943bb1743 100644 --- a/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx +++ b/GPU/GPUTracking/Refit/GPUTrackingRefit.cxx @@ -400,12 +400,11 @@ GPUd() int32_t GPUTrackingRefit::RefitTrack(T& trkX, bool outward, bool resetCov trk.NormalizeAlpha(alpha); prop.SetAlpha(alpha); } else if constexpr (std::is_same_v) { - constexpr float kDeg2Rad = M_PI / 180.f; if (mPparam->rec.tpc.trackReferenceX <= 500) { const int32_t eLossSignRef = (mPparam->rec.tpc.trackReferenceX > trk.getX()) == outward ? eLossSign : -eLossSign; // a forced energy-loss sign holds along the refit direction: the way to the reference X can run against it if (prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef)) { if (CAMath::Abs(trk.getY()) > trk.getX() * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { - float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / kDeg2Rad / 20.f) * GPUTPCGeometry::kSectAngle(); + float newAlpha = trk.getAlpha() + CAMath::Round(CAMath::ATan2(trk.getY(), trk.getX()) / CAMath::Deg2Rad() / 20.f) * GPUTPCGeometry::kSectAngle(); GPUTPCGMTrackParam::NormalizeAlpha(newAlpha); trk.rotate(newAlpha) && prop->PropagateToXBxByBz(trk, mPparam->rec.tpc.trackReferenceX, Propagator::MAX_SIN_PHI, Propagator::MAX_STEP, Propagator::MatCorrType::USEMatCorrLUT, nullptr, eLossSignRef); } diff --git a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h index 0d71b35e92c78..6ef8643215e6d 100644 --- a/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h +++ b/GPU/GPUTracking/TRDTracking/GPUTRDInterfaces.h @@ -147,7 +147,7 @@ class trackInterface : public GPUTPCGMTrackParam GPUd() float getSnp() const { return GetSinPhi(); } GPUd() float getTgl() const { return GetDzDs(); } GPUd() float getQ2Pt() const { return GetQPt(); } - GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * M_PI - CAMath::ATan(getTgl())))); } + GPUd() float getEta() const { return -CAMath::Log(CAMath::Tan(0.5f * (0.5f * CAMath::Pi() - CAMath::ATan(getTgl())))); } GPUd() float getPt() const { return CAMath::Abs(getQ2Pt()) > 0 ? CAMath::Abs(1.f / getQ2Pt()) : 99999.f; } GPUd() float getSigmaY2() const { return GetErr2Y(); } GPUd() float getSigmaZ2() const { return GetErr2Z(); } diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 689f9a36fc2fe..2e8e2c129a1fa 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -182,7 +182,7 @@ static const constexpr int32_t COLORS_HEX[COLORCOUNT] = {0xB03030, 0x00A000, 0x0 static const constexpr int32_t CONFIG_DASHED_MARKERS = 0; static const constexpr float AXES_MIN[5] = {-Y_MAX, -Z_MAX, 0.f, -ETA_MAX, PT_MIN}; -static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * M_PI, ETA_MAX, PT_MAX}; +static const constexpr float AXES_MAX[5] = {Y_MAX, Z_MAX, 2.f * CAMath::Pi(), ETA_MAX, PT_MAX}; static const constexpr int32_t AXIS_BINS[5] = {51, 51, 144, 31, 50}; static const constexpr int32_t RES_AXIS_BINS[] = {1017, 113}; // Consecutive bin sizes, histograms are binned down until the maximum entry is 50, each bin size should evenly divide its predecessor. static const constexpr float RES_AXES[5] = {1., 1., 0.03, 0.03, 1.0}; @@ -1247,12 +1247,12 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx const mcInfo_t& info = GetMCTrack(i, iCol); additionalMCParameters& mc2 = mMCParam[iCol][i]; mc2.pt = std::sqrt(info.pX * info.pX + info.pY * info.pY); - mc2.phi = M_PI + std::atan2(-info.pY, -info.pX); + mc2.phi = CAMath::Pi() + std::atan2(-info.pY, -info.pX); float p = info.pX * info.pX + info.pY * info.pY + info.pZ * info.pZ; if (p < 1e-18) { mc2.theta = mc2.eta = 0.f; } else { - mc2.theta = info.pZ == 0 ? (M_PI / 2) : (std::acos(info.pZ / std::sqrt(p))); + mc2.theta = info.pZ == 0 ? (CAMath::Pi() / 2) : (std::acos(info.pZ / std::sqrt(p))); mc2.eta = -std::log(std::tan(0.5 * mc2.theta)); } if (mConfig.writeMCLabels) { @@ -1306,10 +1306,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } float alpha = std::atan2(info.y, info.x); - alpha /= M_PI / 9.f; + alpha /= CAMath::Pi() / 9.f; alpha = std::floor(alpha); - alpha *= M_PI / 9.f; - alpha += M_PI / 18.f; + alpha *= CAMath::Pi() / 9.f; + alpha += CAMath::Pi() / 18.f; float c = std::cos(alpha); float s = std::sin(alpha); diff --git a/GPU/GPUTracking/qa/genEvents.cxx b/GPU/GPUTracking/qa/genEvents.cxx index 38c2cf2914770..c1f33a1e0bbc9 100644 --- a/GPU/GPUTracking/qa/genEvents.cxx +++ b/GPU/GPUTracking/qa/genEvents.cxx @@ -202,8 +202,8 @@ int32_t genEvents::GenerateEvent(const GPUParam& param, const char* filename) double eta = gRandom->Uniform(-1.5, 1.5); double theta = 2 * std::atan(1. / exp(eta)); - double lambda = theta - M_PI / 2; - // double theta = gRandom->Uniform(-60,60)*M_PI/180.; + double lambda = theta - CAMath::Pi() / 2; + // double theta = gRandom->Uniform(-60,60)*CAMath::Pi()/180.; double pt = .08 * std::pow(10, gRandom->Uniform(0, 2.2)); double q = 1.; diff --git a/GPU/GPUTracking/qa/genEvents.h b/GPU/GPUTracking/qa/genEvents.h index ea209a0a46cb8..6bd6d4b370f64 100644 --- a/GPU/GPUTracking/qa/genEvents.h +++ b/GPU/GPUTracking/qa/genEvents.h @@ -16,7 +16,7 @@ #define GENEVENTS_H #include "GPUCommonDef.h" -#include +#include "GPUCommonMath.h" namespace o2::gpu { @@ -66,7 +66,7 @@ class genEvents uint32_t id; }; - const double mTwoPi = 2 * M_PI; + const double mTwoPi = 2 * CAMath::Pi(); const double mSectorDAngle = mTwoPi / 18.; const double mSectorAngleOffset = mSectorDAngle / 2; From 60516412567a795dd7e014d1c920f4d4024c574e Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 00:11:27 +0100 Subject: [PATCH 14/54] GPU TPC: Store track position for interpolation also when not updating with new hit --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 133 ++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 4 +- 4 files changed, 76 insertions(+), 65 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index 49e0773abab1d..ddfefe8636af5 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -638,7 +638,7 @@ GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int32_t iRow, return Update(posY, posZ, clusterState, rejectChi2, err2Y, err2Z, ¶m); } -GPUd() void GPUTPCGMPropagator::InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter) +GPUd() void GPUTPCGMPropagator::InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter) { float* GPUrestrict() mC = mT->Cov(); float* GPUrestrict() mP = mT->Par(); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index bf67d027f6b05..e637c315f3ce6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -102,7 +102,7 @@ class GPUTPCGMPropagator GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, int8_t sector, float time, float avgInvCharge, float invCharge); GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); - GPUd() void InterpolateFill(float posY, float posZ, gputpcgmmergertypes::InterpolationErrorHit* inter); + GPUd() void InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter); GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index ee70dd488a44e..7cc312fcdd4f2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -123,10 +123,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); + // TODO: Check about tracks who have clusters in the same row multiple times in different sectors const auto& cluster = clusters[ihit]; + CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { @@ -140,31 +143,67 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); - FitAddRow(iRow, cluster.sector, iTrk, track, tmpX, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { + break; + } + FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; } lastPropagateRow = cluster.row; - if (currentClusterStatus) { - nMissed++; + int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); + if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha + (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all + prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection))) || // propagation fails nonetheless + retValProp) { // failed for other reason but rotation + CADEBUG(printf(" --- break-prop\n")); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - if (doInterpolate) { - FitAddRow(cluster.row, cluster.sector, iTrk, track, xx, prop, inFlyDirection, merger, nullptr, false, false, true); - } + NTolerated++; continue; } - - CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + // clang-format off + CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); + CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); + // clang-format on + if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + break; // bad chi2 for the whole track, stop the fit + } if ((uint32_t)interpolationIndex >= interpolation.size) { merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } auto& inter = interpolation.hit[interpolationIndex]; + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + + float uncorrectedY = -1e6f; + if (param.rec.tpc.rebuildTrackInFit) { + if (iWay == nWays - 2) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } + if (allowChangeClusters) { + AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit + } + } else if (allowChangeClusters) { + uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + float tmpZ; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); + } + + if (currentClusterStatus) { + nMissed++; + nMissed2++; + continue; + } + const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -226,11 +265,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float tmpX, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) { - if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { - return; - } if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; } @@ -250,46 +286,20 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - // clang-format off - CADEBUG(if (!CheckCov()){printf("INVALID COV AFTER PROPAGATE!!!\n");}); - // clang-format on - if (retValProp == -2) // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha - { - CADEBUG(printf("REROTATE\n")); - if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) == 0) { - retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); - } - } - // clang-format off - CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); - // clang-format on - - float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - const bool sinPhiErr = mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate; - if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - return 1; // bad chi2 for the whole track, stop the fit - } - if (retValProp || sinPhiErr) { + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; - CADEBUG(printf(", %d --- break\n", (int32_t)sinPhiErr)); + CADEBUG(printf(" --- break-sinphi\n")); return 2; // Propagate failed or high incl angle } - CADEBUG(printf("\n")); if (crossCE) { if (param.rec.tpc.addErrorsCECrossing) { @@ -307,6 +317,9 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + if (param.rec.tpc.rebuildTrackInFit) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds + } } else { float err2Y, err2Z; const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[cluster.num].getTime() : -1.f; @@ -317,15 +330,15 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = false; + bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 3) { - prop.InterpolateFill(yy, zz, &inter); - } else if (iWay == nWays - 2) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else if (!param.rec.tpc.rebuildTrackInFit) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + rejectChi2 = true; + } else { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } } } else if (iWay == nWays - 1) { if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { @@ -333,10 +346,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } } else { - rejectChi2 = GetNDF() > 0 && allowChangeClusters; + rejectChi2 = allowChangeClusters; } - if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { // uncorrectedY > -1e6f implies allowChangeClusters + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedEdge; } else { retValUpd = prop.Update(yy, zz, cluster.row, param, clusterState, rejectChi2, refit, err2Y, err2Z); @@ -367,7 +380,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (retValInt || allowChangeClusters) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); } else if (finalFit) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected ? GPUTPCGMMergedTrackHit::flagRejectDistance : GPUTPCGMMergedTrackHit::flagRejectErr); } if (!retValInt) { nMissed++; @@ -473,7 +486,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("iTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } if (nCandidates == 0) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; @@ -599,17 +612,17 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG(, this)](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { - CADEBUG(printf("\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); + CADEBUG(printf("\t\t\tRejecting double-row cluster: dy %f, dz %f, chiY %f, chiZ %f (Y: trk %f prj %f cl %f - Z: trk %f prj %f cl %f)\n", dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2), mP[0], projY, y, mP[1], projZ, z)); if (markReject) { clusters[ih].state |= GPUTPCGMMergedTrackHit::flagRejectDistance; } return false; } else { - CADEBUG(printf("\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); + CADEBUG(printf("\t\t\tMerging hit row %d Y %f Z %f (dy %f, dz %f, chiY %f, chiZ %f)\n", clusters[ih].row, y, z, dy, dz, sqrtf(maxDistY2), sqrtf(maxDistZ2))); return true; } }; @@ -1061,9 +1074,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - // clang-format off - CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; } ); - // clang-format on + CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index f0ce761d41418..29b4f446965b4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,8 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float tmpX, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From af80e5e4fc12d9337a436f65442b3acb54538c91 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 10:39:27 +0100 Subject: [PATCH 15/54] GPU TPC: Extract CE crossing handling in function, and apply before storing interpolated position --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 +++++++++++-------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7cc312fcdd4f2..eebef43473413 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -97,7 +97,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = 0; - uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastUpdateSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -133,7 +133,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastUpdateSector && currentClusterStatus == 0; + bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); if (dodEdx || doAttach || doInterpolate) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; @@ -177,9 +177,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } auto& inter = interpolation.hit[interpolationIndex]; - if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { - prop.InterpolateFill(&inter); - } float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { @@ -196,14 +193,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } + HandleCrossCE(param, cluster.sector, lastSector); + lastSector = cluster.sector; + + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + prop.InterpolateFill(&inter); + } + if (currentClusterStatus) { nMissed++; nMissed2++; continue; } - const bool crossCE = lastUpdateSector != 255 && ((lastUpdateSector < 18) ^ (clusters[ihit].sector < 18)); - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, crossCE, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -218,11 +221,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastUpdateRow = cluster.row; - lastUpdateSector = cluster.sector; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { - StoreLoopPropagation(merger, lastUpdateSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { @@ -286,7 +288,23 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) +{ + const bool crossCE = lastSector != 255 && ((lastSector < 18) ^ (sector < 18)); + if (crossCE) { + if (param.rec.tpc.addErrorsCECrossing) { + if (param.rec.tpc.addErrorsCECrossing >= 2) { + AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); + } else { + AddCovDiagErrors(param.rec.tpc.errorsCECrossing); + } + } else if (mC[2] < 0.5f) { + mC[2] = 0.5f; + } + } +} + +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -301,18 +319,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger return 2; // Propagate failed or high incl angle } - if (crossCE) { - if (param.rec.tpc.addErrorsCECrossing) { - if (param.rec.tpc.addErrorsCECrossing >= 2) { - AddCovDiagErrorsWithCorrelations(param.rec.tpc.errorsCECrossing); - } else { - AddCovDiagErrors(param.rec.tpc.errorsCECrossing); - } - } else if (mC[2] < 0.5f) { - mC[2] = 0.5f; - } - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 29b4f446965b4..c6f2d58cc9cca 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,8 +156,9 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, const bool crossCE, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); From 39e82a394225b98ffd59e73aadc91f2d9dc2aac4 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 11:16:51 +0100 Subject: [PATCH 16/54] GPU TPC: If value from other side of interpolation not stored, use only current value --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 3 +- GPU/GPUTracking/Global/GPUChainTracking.cxx | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 36 +++++++++++-------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index cac6faf8e0908..e07ac4f379434 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -102,6 +102,8 @@ AddOptionRTC(rejectEdgeClustersMargin, float, 0.f, "", 0, "Margin in cm of Y pos AddOptionRTC(rejectEdgeClustersSigmaMargin, float, 0.f, "", 0, "Margin factor for trackSigmaY when rejecting edge clusters based on uncorrected track Y") AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for tracklet") AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") +AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") +AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(maxTimeBinAboveThresholdIn1000Bin, uint16_t, 500, "", 0, "Except pad from cluster finding if total number of charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(maxConsecTimeBinAboveThreshold, uint16_t, 200, "", 0, "Except pad from cluster finding if number of consecutive charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(noisyPadSaturationThreshold, uint16_t, 700, "", 0, "Threshold where a timebin is considered saturated, disabling the noisy pad check for that pad") @@ -137,7 +139,6 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 8ab663b5f791a..b3c1d02e99c8d 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1031,6 +1031,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index eebef43473413..98e4112cea6a8 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -406,7 +406,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -414,19 +414,24 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre const float stepZ = rowData.HstepZ(); int32_t bin, ny, nz; - float err2Y, err2Z; - param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge - - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; + float ImP0, ImP1, ImC0, ImC2; + if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = inter.posY - mP[0]; + const float Iz1 = inter.posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0; + ImP1 = mP[1] + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0]; + ImC2 = mC[2] - Ik11 * mC[2]; + } else { + ImP0 = mP[0]; + ImP1 = mP[1]; + ImC0 = mC[0]; + ImC2 = mC[2]; + } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); @@ -435,6 +440,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); From a427f2b37d02a2f4d0a46fcdb8d36145ee271944 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 00:00:45 +0100 Subject: [PATCH 17/54] GPU TPC: Mark clusters of rebuilt track as high incli when interpolated in NoUpdateHighIncl region --- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 6 ++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 5 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 37 +++++++++++++------ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index ef5abc7e49a31..be9f447093e12 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2271,7 +2271,11 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads } const auto& best = candidates[candidates[0].best - 1]; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; - outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags)}; + uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); + if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) { + flags |= GPUTPCGMMergedTrackHit::flagHighIncl; + } + outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags}; written++; CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index d07a47d4fe981..66ff40f79ac3f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -88,7 +88,9 @@ class GPUTPCGMMerger : public GPUProcessor }; struct trackRebuildHelper { - bool reverse; + uint8_t reverse; + uint8_t highInclRowLow; + uint8_t highInclRowHigh; }; struct tmpSort { @@ -133,6 +135,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 98e4112cea6a8..5792987379458 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -64,6 +64,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size interpolation.hit[i].errorY = -1; } + if (param.rec.tpc.rebuildTrackInFit) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + } } const int32_t nWays = param.rec.tpc.nWays; @@ -206,6 +210,28 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + const bool inward = clusters[0].row > clusters[maxN - 1].row; + const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } + } + } + continue; + } + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); @@ -215,9 +241,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ covYYUpd = mC[0]; } else if (retValHit == 1) { break; - } else if (retValHit == 2) { - NTolerated++; - continue; } lastUpdateRow = cluster.row; @@ -311,14 +334,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger const int32_t wayDirection = (iWay & 1) ? -1 : 1; const auto& cluster = clusters[ihit]; - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - return 2; // Propagate failed or high incl angle - } - int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { From bdf7c31594d2326e6770b4422602e8c56fd2f684 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 30 Oct 2025 18:39:10 +0100 Subject: [PATCH 18/54] GPU TPC: Pick up best hit from interpolation only, if current position not available --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 54 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 3 +- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index b3c1d02e99c8d..df74187a4f9d5 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -1031,7 +1031,7 @@ void GPUChainTracking::ApplySyncSettings(GPUSettingsProcessing& proc, GPUSetting { if (syncMode) { rec.useMatLUT = false; - rec.tpc.rebuildTrackMaxNonIntCov = 0.f; + rec.tpc.rebuildTrackMaxNonIntCov = 0.f; // TODO: Check if this yields a performance benefit } if (proc.rtc.optSpecialCode == -1) { proc.rtc.optSpecialCode = syncMode; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 5792987379458..2ec863fdb6861 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -106,7 +106,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart - wayDirection; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); @@ -123,7 +123,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool allowChangeClusters = finalOutInFit && (nWays == 1 || ((iWay & 1) ? (ihit <= CAMath::Max(maxN / 2, maxN - 30)) : (ihit >= CAMath::Min(maxN / 2, 30)))); int32_t ihitMergeFirst = ihit; - interpolationIndex += wayDirection; uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; @@ -157,6 +156,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastPropagateRow = cluster.row; + auto& inter = interpolation.hit[interpolationIndex]; int32_t retValProp = prop.PropagateToXAlpha(xx, clAlpha, inFlyDirection); if ((retValProp == -2 && // Rotation failed, try to bring to new x with old alpha first, rotate, and then propagate to x, alpha (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection) != 0 || // Cannot rotate to new alpha at all @@ -166,6 +166,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later + } continue; } // clang-format off @@ -173,6 +176,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; // bad chi2 for the whole track, stop the fit } @@ -180,12 +186,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); break; } - auto& inter = interpolation.hit[interpolationIndex]; float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit @@ -240,6 +245,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + } break; } @@ -413,7 +421,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -421,7 +429,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -430,9 +438,14 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre int32_t bin, ny, nz; float ImP0, ImP1, ImC0, ImC2; - if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - const float Iz0 = inter.posY - mP[0]; - const float Iz1 = inter.posZ + deltaZ - mP[1]; + if (interOnly) { + ImP0 = (float)inter.posY; + ImP1 = (float)inter.posZ + deltaZ; + ImC0 = (float)inter.errorY; + ImC2 = (float)inter.errorZ; + } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + const float Iz0 = (float)inter.posY - mP[0]; + const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); const float Iw2 = 1.f / (mC[2] + (float)inter.errorZ); const float Ik00 = mC[0] * Iw0; @@ -517,14 +530,33 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); } - if (nCandidates == 0) { + if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; } } return uncorrectedY; } -GPUdii() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +{ + for (; ihit >= 0; ihit--, interpolationIndex--) { + while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { + ihit--; + } + const auto& cluster = clusters[ihit]; + if (CAMath::Abs(cluster.row - lastRow) > 1) { + interpolationIndex -= CAMath::Abs(cluster.row - lastRow) - 1; + } + lastRow = cluster.row; + if (interpolationIndex < 0) { // TODO: Is this check needed? + return; + } + auto inter = interpolation.hit[interpolationIndex]; + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + } +} + +GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) { const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index c6f2d58cc9cca..572dfe67c2330 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,8 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); From 65c11529ca4608aa15009050c99cc4ac837960db Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 13:04:02 +0100 Subject: [PATCH 19/54] GPU TPC: Interpolate missing pad rows when rebuilding track --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 50 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index d8b14ecc61182..88388a4b9074f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -35,6 +35,8 @@ enum attachTypes { attachProtect = 0x80000000, struct InterpolationErrorHit { float posY, posZ; GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; + GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } + GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2ec863fdb6861..47dcf98f583d3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -61,12 +61,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing size - interpolation.hit[i].errorY = -1; + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range + interpolation.hit[i].markInvalid(); } if (param.rec.tpc.rebuildTrackInFit) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = 255; - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; + merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; } } @@ -134,22 +133,24 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); uint8_t dEdxSubThresholdRow = 255; - bool doInterpolate = param.rec.tpc.rebuildTrackInFit && (iWay == nWays - 3 || iWay == nWays - 2); if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { - bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - if (dodEdx || doAttach || doInterpolate) { + const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); + const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; - for (int32_t iRow = lastPropagateRow + step; iRow != cluster.row; iRow += step) { + uint8_t sector = lastSector; + + for (int32_t iRow = lastPropagateRow + step, index = interpolationIndex; iRow != cluster.row; iRow += step, index += wayDirection) { float tmpX, tmpY, tmpZ; - if (prop.GetPropagatedYZ(mX - GPUTPCGeometry::Row2X(iRow - step) + GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; } - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(cluster.sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, prop.GetAlpha(), inFlyDirection)) { break; } - FitAddRow(iRow, cluster.sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate); + FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); } } interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; @@ -233,6 +234,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; } } + // TODO: We can perhaps break here, if we pick up remaining rows } continue; } @@ -298,7 +300,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } -GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate) +GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { return; @@ -317,6 +319,16 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se if (doAttach) { AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); } + if (doInterpolate) { + if (doInterpolate == 2) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > 4.f) { + inter.markInvalid(); + } + FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + } else { + prop.InterpolateFill(&inter); + } + } } GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() param, const uint8_t sector, const uint8_t& lastSector) @@ -363,10 +375,10 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (param.rec.tpc.mergerInterpolateErrors) { if (iWay == nWays - 2) { if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.errorY < (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { - rejectChi2 = true; - } else { + if (inter.isValid()) { retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; } } } else if (iWay == nWays - 1) { @@ -429,7 +441,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0 || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + if (rowData.NHits() && (inter.isValid() || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -443,7 +455,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre ImP1 = (float)inter.posZ + deltaZ; ImC0 = (float)inter.errorY; ImC2 = (float)inter.errorZ; - } else if (inter.errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0) { + } else if (inter.isValid()) { const float Iz0 = (float)inter.posY - mP[0]; const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); @@ -520,7 +532,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (insert < param.rec.tpc.rebuildTrackInFitClusterCandidates) { for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; - } + } // TODO: Downweight non-interpolated candidates merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 572dfe67c2330..7371cfd8afdef 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -157,7 +157,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); - GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const bool doInterpolate); + GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); From 0199544ffcc14e84c2d5efa5daeed5fc528e78c9 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 31 Oct 2025 14:07:16 +0100 Subject: [PATCH 20/54] Revert "GPU TPC: Remove option to retry refit if cluster rejection breaks the track, we will anyway rebuild the track in the future" This reverts commit 5c3d113f80687ecfcb3bf106b0fc7ad274fceda9. We can actually properly retry also for rebuilt tracks, we just run the retry on the last iteration after rebuilding. --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 4 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 5 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 + GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 7 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 62 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 6 +- GPU/GPUTracking/qa/GPUQA.cxx | 2 +- 8 files changed, 60 insertions(+), 30 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index e07ac4f379434..f190cd11036b1 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -157,6 +157,7 @@ AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation inst AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") +AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 52cdcd5dee78f..596bc8f614666 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -248,6 +248,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); } + if (param().rec.tpc.retryRefit) { + runKernel(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit); + } + runKernel(GetGridAuto(0)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRefit, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index be9f447093e12..8b4bf329634cb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -402,13 +402,16 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mTrackSort, mNMaxTracks); memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; - computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks for mLoopData does not save memory, other parts are larger anyway + computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { computePointerWithAlignment(mem, mSharedCount2, mNClusters); computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); computePointerWithAlignment(mem, mHitWeights, mNClusters); } + if (mRec->GetParam().rec.tpc.retryRefit) { + computePointerWithAlignment(mem, mRetryRefitIds, mNMaxTracks); + } memMax = (void*)std::max((size_t)mem, (size_t)memMax); mem = memBase; computePointerWithAlignment(mem, mLooperCandidates, mNMaxLooperMatches); // MergeLoopers 1-3 diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 66ff40f79ac3f..ee63269b532fb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -67,6 +67,7 @@ class GPUTPCGMMerger : public GPUProcessor static GPUglobalconstexpr() const int32_t NSECTORS = GPUTPCGeometry::NSECTORS; //* N sectors struct memory { + GPUAtomic(uint32_t) nRetryRefit; GPUAtomic(uint32_t) nLoopData; GPUAtomic(uint32_t) nUnpackedTracks; GPUAtomic(uint32_t) nMergedTracks; @@ -140,6 +141,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } GPUhdi() uint32_t* TrackOrderAttach() const { return mTrackOrderAttach; } GPUhdi() uint32_t* TrackOrderProcess() const { return mTrackOrderProcess; } + GPUhdi() uint32_t* RetryRefitIds() const { return mRetryRefitIds; } GPUhdi() uint8_t* ClusterStateExt() const { return mClusterStateExt; } GPUhdi() GPUTPCGMLoopData* LoopData() const { return mLoopData; } GPUhdi() memory* Memory() const { return mMemory; } @@ -322,6 +324,7 @@ class GPUTPCGMMerger : public GPUProcessor gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRangeMemory = nullptr; // memory for border tracks gputpcgmmergertypes::GPUTPCGMBorderRange* mBorderRange[NSECTORS]; // memory for border tracks memory* mMemory = nullptr; + uint32_t* mRetryRefitIds = nullptr; GPUTPCGMLoopData* mLoopData = nullptr; }; } // namespace o2::gpu diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 083cdf76699a0..2ef0d0ba59748 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -21,9 +21,10 @@ using namespace o2::gpu; template <> GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) { - GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, merger.NMergedTracks(), { - const int32_t i = mode ? merger.TrackOrderProcess()[ii] : ii; - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt); + const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); + GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { + const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); + GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1); }); } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 47dcf98f583d3..75e8786ff9186 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -47,7 +47,7 @@ using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt) +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; @@ -91,7 +91,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool finalFit = iWay == nWays - 1; ResetCovariance(); - prop.SetSeedingErrors(!(refit)); + prop.SetSeedingErrors(!(refit && !retryAttempt)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); @@ -239,7 +239,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); ihitStart = ihit; @@ -256,7 +256,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255) { + if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -347,7 +347,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -371,23 +371,25 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, time, invAvgCharge, invCharge); - bool rejectChi2 = (clusterState & GPUTPCGMMergedTrackHit::flagReject); - if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 2) { - if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); - } else { + bool rejectChi2 = !retryAttempt && (clusterState & GPUTPCGMMergedTrackHit::flagReject); + if (!retryAttempt) { + if (param.rec.tpc.mergerInterpolateErrors) { + if (iWay == nWays - 2) { + if (!param.rec.tpc.rebuildTrackInFit) { + if (inter.isValid()) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; + } + } + } else if (iWay == nWays - 1) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { rejectChi2 = true; } } - } else if (iWay == nWays - 1) { - if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { - rejectChi2 = true; - } + } else { + rejectChi2 = allowChangeClusters; } - } else { - rejectChi2 = allowChangeClusters; } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY > -1e6f && param.rejectEdgeClusterByY(uncorrectedY, cluster.row, CAMath::Sqrt(mC[0]))) { @@ -893,6 +895,10 @@ GPUdic(0, 1) void GPUTPCGMTrackParam::StoreLoopPropagation(const GPUTPCGMMerger& GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t loopIdx) { + GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; + if (!merger.MergedTracks()[data.track].OK()) { + return; + } GPUTPCGMPropagator prop; prop.SetMaterialTPC(); prop.SetPolynomialField(&merger.Param().polynomialField); @@ -902,7 +908,6 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr prop.SetFitInProjections(true); prop.SetPropagateBzOnly(false); - GPUTPCGMLoopData& data = merger.LoopData()[loopIdx]; prop.SetTrack(&data.param, data.alpha); if (merger.Param().rec.tpc.looperFollowMode == 1) { data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); @@ -1141,7 +1146,7 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt) // VS: GPUd changed to GPUdii. No change in output and no performance penalty. +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt) { if (!track.OK()) { return; @@ -1153,16 +1158,29 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); + if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { + for (uint32_t i = 0; i < track.NClusters(); i++) { + merger.Clusters()[track.FirstClusterRef() + i].state &= GPUTPCGMMergedTrackHit::clustererAndSharedFlags; + } + CADEBUG(printf("Track rejected, marking for retry\n")); + uint32_t nRefit = CAMath::AtomicAdd(&merger.Memory()->nRetryRefit, 1u); + merger.RetryRefitIds()[nRefit] = iTrk; + return; + } + if (retryAttempt && (t.mNDF < 10 || t.mChi2 / t.mNDF >= 6.f)) { + ok = false; + } + if (CAMath::Abs(t.QPt()) < 1.e-4f) { t.QPt() = CAMath::Copysign(1.e-4f, t.QPt()); } CADEBUG(if (t.GetX() > 250) { printf("ERROR, Track %d at impossible X %f, Pt %f, Looper %d\n", iTrk, t.GetX(), CAMath::Abs(1.f / t.QPt()), (int32_t)merger.MergedTracks()[iTrk].Looper()); }); - track.SetOK(ok); // TODO: Should we recover tracks who failed the fit in iWay0/1 for the rebuild? + track.SetOK(ok); if (t.GetNDF() <= 0 && !rebuilt && merger.Param().rec.tpc.rebuildTrackInFit) { // TODO: Better handling of NDF<0 tracks, how do we want to do cluster rejection? track.Param().NDF() = 0; } else { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 7371cfd8afdef..a2a677db8c5de 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -154,12 +154,12 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt); + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt); + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 2e8e2c129a1fa..2038c4c84f707 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2968,7 +2968,7 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } } - if (mQATasks & taskTrackStatistics) { + if (mQATasks & taskTrackStatistics) { // TODO: Add plot for Chi2/NDF, and NCl correct and fake MC histograms // Process track statistic histograms float tmpMax = 0.; From 8186efebe295962c8ff76b1a65612987f75cf1cc Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 11:06:13 +0100 Subject: [PATCH 21/54] GPU TPC: Relax chi2cut for single-pad clusters --- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index e637c315f3ce6..3f1a74cfb6c26 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -108,13 +108,19 @@ class GPUTPCGMPropagator GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) { - if (chiY > 9.f || chiZ > 9.f) { // TODO: Check how a track can have chi2/ncl > 18 + if (chiY > 9.f || chiZ > 9.f) { return 2; } - if ((chiY > 6.25f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + if (chiZ > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared | GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { return 2; } - if ((chiY > 1.f || chiZ > 6.25f) && (clusterState & (GPUTPCGMMergedTrackHit::flagEdge | GPUTPCGMMergedTrackHit::flagSingle))) { + if (chiY > 6.25f && (clusterState & (GPUTPCGMMergedTrackHit::flagSplit | GPUTPCGMMergedTrackHit::flagShared))) { + return 2; + } + if (chiY > 1.f && (clusterState & GPUTPCGMMergedTrackHit::flagEdge)) { + return 2; + } + if (chiY > 2.f && (clusterState & GPUTPCGMMergedTrackHit::flagSingle)) { return 2; } return 0; From f6cc1880ac213ba79b9a89206604b1497aeef0e2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:41:14 +0100 Subject: [PATCH 22/54] GPU TPC: Clean up and fix resetT0 code --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 22 +++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 15 ++----------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 75e8786ff9186..9bce56b33b057 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -85,7 +85,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ StoreOuter(&track.OuterParam(), prop.GetAlpha()); // TODO: Fix outer param! } - int32_t resetT0 = initResetT0(); + int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; @@ -239,9 +239,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, resetT0, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); if (retValHit == 0) { - DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow); + DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + resetT0 = 20; + } + ihitStart = ihit; interpolatedStart = interpolationIndex; N++; @@ -347,7 +353,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, int32_t& GPUrestrict() resetT0, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -412,12 +418,6 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger lastUpdateX = mX; nMissed = nMissed2 = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - float dy = mP[0] - prop.Model().Y(); - float dz = mP[1] - prop.Model().Z(); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && dy * dy + dz * dz > 1) { - CADEBUG(printf("Reinit linearization\n")); - prop.SetTrack(this, prop.GetAlpha()); - } return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track if (retValInt || allowChangeClusters) { @@ -570,7 +570,7 @@ GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrest } } -GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow) +GPUd() void GPUTPCGMTrackParam::DodEdx(GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, GPUTPCGMMerger& GPUrestrict() merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* GPUrestrict() clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)) { const GPUParam& GPUrestrict() param = merger.Param(); if GPUCA_RTC_CONSTEXPR (GPUCA_GET_CONSTEXPR(param.par, dodEdx)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index a2a677db8c5de..6b5322a8ec1cf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -155,8 +155,8 @@ class GPUTPCGMTrackParam GPUd() bool CheckCov() const; GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); - GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, int32_t& resetT0, float uncorrectedY, bool retryAttempt); + GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); @@ -229,8 +229,6 @@ class GPUTPCGMTrackParam } private: - GPUd() int32_t initResetT0(); - float mX; // x position float mTOffset; // Z offset with early transform, T offset otherwise float mP[5]; // 'active' track parameters: Y, Z, SinPhi, DzDs, q/Pt @@ -248,15 +246,6 @@ struct GPUTPCGMLoopData { uint8_t outwards; }; -GPUdi() int32_t GPUTPCGMTrackParam::initResetT0() -{ - const float absQPt = CAMath::Abs(mP[4]); - if (absQPt < (150.f / 40.f)) { - return 150.f / 40.f; - } - return CAMath::Max(10.f, 150.f / mP[4]); -} - GPUdi() void GPUTPCGMTrackParam::ResetCovariance() { mC[0] = 100.f; From cbe59f3af769e7e84d37ab1033797dfa52e0d682 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 15:42:27 +0100 Subject: [PATCH 23/54] GPU TPC: Don't break for high sinPhi if only the linearization model is out of range, but instead reinitialize the model --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 9bce56b33b057..2e499bd501b5b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -217,26 +217,31 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); - if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { - MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; - CADEBUG(printf(" --- break-sinphi\n")); - NTolerated++; + if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); - if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { - if (inward ^ (iWay & 1)) { - if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; - } - } else { - if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { - merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + if (mNDF > 10 && CAMath::Abs(mP[2]) < maxSinForUpdate && markHighIncl) { + CADEBUG(printf("Reinit linearization\n")); + prop.SetTrack(this, prop.GetAlpha()); + } else { + MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); + nMissed2++; + CADEBUG(printf(" --- break-sinphi\n")); + NTolerated++; + if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { + if (inward ^ (iWay & 1)) { + if (merger.TrackRebuildHelper()[iTrk].highInclRowLow == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = cluster.row; + } + } else { + if (merger.TrackRebuildHelper()[iTrk].highInclRowHigh == 255) { + merger.TrackRebuildHelper()[iTrk].highInclRowHigh = cluster.row; + } } + // TODO: We can perhaps break here, if we pick up remaining rows } - // TODO: We can perhaps break here, if we pick up remaining rows + continue; } - continue; } int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); From cc02414ea97caf9b9249409e2c8676ec50b18506 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 18 Nov 2025 15:37:23 +0100 Subject: [PATCH 24/54] GPU: Add option to steer which attachment steps to run during rebuild --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 7 +++--- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 22 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 +++++++++--------- .../Standalone/Benchmark/standalone.cxx | 2 +- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index f190cd11036b1..b5c2f7825a361 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -139,13 +139,14 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") -AddOptionRTC(disableRefitAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during refit (1: attachment, 2: propagation, 4: loop following)") -AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableRefitAttachment = 4") +AddOptionRTC(looperFollowMode, uint8_t, 1, "", 0, "0 = simple, 1 = advances, for disabling use disableMarkAdjacent = 4") AddOptionRTC(rejectionStrategy, uint8_t, o2::gpu::GPUSettings::RejectionStrategyA, "", 0, "Enable rejection of TPC clusters for compression (0 = no, 1 = strategy A, 2 = strategy B)") AddOptionRTC(mergeLoopersAfterburner, uint8_t, 1, "", 0, "Run afterburner for additional looper merging") AddOptionRTC(compressionTypeMask, uint8_t, o2::gpu::GPUSettings::CompressionFull, "", 0, "TPC Compression mode bits (1=truncate charge/width LSB, 2=differences, 4=track-model)") @@ -168,7 +169,7 @@ AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge cluste AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") -AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet hand AddOptionArrayRTC +AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 8b4bf329634cb..5b71478facb0d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2145,16 +2145,18 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, continue; } mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; - int lastRow = -1; - for (uint32_t j = 0; j < trk.NClusters(); j++) { - const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { - candidates[0].id = cl.num + 2; - candidates[0].best = 128; - candidates[0].weight = cl.state; - candidates[0].sector = cl.sector; - lastRow = cl.row; + if (!(Param().rec.tpc.disableRebuildAttachment & 8)) { + int lastRow = -1; + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { + candidates[0].id = cl.num + 2; + candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly + candidates[0].weight = cl.state; + candidates[0].sector = cl.sector; + lastRow = cl.row; + } } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2e499bd501b5b..46f3562177eb0 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -135,8 +135,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableRefitAttachment & 2); - const uint8_t doInterpolate = (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0); + const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); + const uint8_t doInterpolate = (param.rec.tpc.disableRebuildAttachment & 2) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -167,7 +167,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later } continue; @@ -177,7 +177,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; // bad chi2 for the whole track, stop the fit @@ -190,15 +190,16 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == nWays - 2) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps } if (allowChangeClusters) { AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit } } else if (allowChangeClusters) { uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); - } else if (param.rec.tpc.rejectEdgeClustersInTrackFit) { + } + if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { float tmpZ; merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(cluster.sector, cluster.row, mP[0], mP[1], uncorrectedY, tmpZ); } @@ -258,7 +259,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N++; covYYUpd = mC[0]; } else if (retValHit == 1) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0) { + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } break; @@ -267,7 +268,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableRefitAttachment & 4) && lastUpdateRow != 255 && !retryAttempt) { + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -771,7 +772,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { const GPUParam& GPUrestrict() param = merger.Param(); - if (param.rec.tpc.disableRefitAttachment & 1) { + if (param.rec.tpc.disableMarkAdjacent & 1) { return -1e6f; } const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 38ca9adb3d242..652ec5ccc23f2 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -538,7 +538,7 @@ int32_t SetupReconstruction() if (recSet.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { procSet.tpcInputWithClusterRejection = 1; } - recSet.tpc.disableRefitAttachment = 0xFF; + recSet.tpc.disableMarkAdjacent = 0xFF; recSet.maxTrackQPtB5 = CAMath::Min(recSet.maxTrackQPtB5, recSet.tpc.rejectQPtB5); GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, false, configStandalone.rundEdx); recAsync->SetSettings(&grp, &recSet, &procSet, &steps); From 57f4f593cb70c0da41b997506257481154974ecb Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 2 Nov 2025 19:19:38 +0100 Subject: [PATCH 25/54] GPU TPC: Extrapolate track inward and outward when rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 6 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 114 +++++++++++++++++- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index b5c2f7825a361..92e4122fdf649 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -139,7 +139,7 @@ AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") -AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit)") +AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") @@ -168,7 +168,9 @@ AddOptionRTC(rejectEdgeClustersInSeeding, int8_t, 0, "", 0, "Reject edge cluster AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge clusters based on uncorrected track Y during track fit") AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") - +AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 15, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 6, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 46f3562177eb0..c40236dca1b71 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -50,6 +50,7 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; + const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const GPUParam& GPUrestrict() param = merger.Param(); GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -217,7 +218,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ continue; } - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); if (mNDF > 0 && CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate) { // TODO: If NDF is large enough, and mP[2] is not yet out of range, reinit linearization const bool inward = clusters[0].row > clusters[maxN - 1].row; const bool markHighIncl = (mP[2] > 0) ^ (mP[4] < 0) ^ inward ^ (iWay & 1); @@ -268,16 +268,120 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ lastUpdateRow = cluster.row; assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); } - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { - StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); - CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); - } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); } else { deltaZ = 0.f; } + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; + int32_t sector = lastSector; + uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; + uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; + uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; + prop.SetTrack(this, prop.GetAlpha()); + for (uint32_t iRow = lastPropagateRow + up; iRow < GPUTPCGeometry::NROWS; iRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + bool fail = false; + for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { + float tmpX, tmpY, tmpZ; + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + fail = true; + break; + } + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); + if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { + fail = true; + break; + } + if (CAMath::Abs(mP[2]) > maxSinForUpdate) { + fail = true; + break; + } + if (CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f) + 0.1f) { + if (iAttempt) { + fail = true; + break; + } + const int32_t sectorSide = sector >= (int8_t)(GPUTPCGeometry::NSECTORS / 2) ? (GPUTPCGeometry::NSECTORS / 2) : 0; + if (mP[0] > 0) { + if (++sector >= sectorSide + 18) { + sector -= 18; + } + } else { + if (--sector < sectorSide) { + sector += 18; + } + } + } + } + if (fail) { + break; + } + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + gputpcgmmergertypes::InterpolationErrorHit inter; + inter.markInvalid(); + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + if (candidate.id >= 2) { + lastExtrapolateRow = iRow; + float err2Y, err2Z, xx, yy, zz; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; + merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { + candidate.best = -1; + break; + } + const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? cl.getTime() : -1.f; // TODO: When is it possible that we do not have clusterNative? + const float invSqrtCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? CAMath::InvSqrt(cl.qMax) : 0.f; + const float invCharge = merger.GetConstantMem()->ioPtrs.clustersNative ? (1.f / cl.qMax) : 0.f; + float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; + invAvgCharge *= invAvgCharge; + const uint8_t clusterState = cl.getFlags(); + prop.GetErr2(err2Y, err2Z, param, zz, iRow, clusterState, sector, time, invAvgCharge, invCharge); + CADEBUG(printf("\t%21sResiduals %8.3f %8.3f --- Errors %8.3f %8.3f\n", "", yy - mP[0], zz - mP[1], CAMath::Sqrt(err2Y), CAMath::Sqrt(err2Z))); + uint32_t retValUpd = prop.Update(yy, zz, iRow, param, clusterState, true, refit, err2Y, err2Z); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (retValUpd != 0) { + candidate.best = -1; + break; + } + if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { + lastGoodRow = iRow; + consecGoodRowsMissing = 0; + } + rowGapActive = rowGapTotal = 0; + } else { + if (++missingRowsTotal > param.rec.tpc.rebuildTrackExtrMaxMissingRows) { + break; + } + if (++rowGapTotal > param.rec.tpc.trackFollowingMaxRowGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); + if (pad < GPUTPCGeometry::NPads(iRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + if (rowGapActive++ >= param.rec.tpc.trackFollowingMaxRowGap) { + break; + } + if (consecGoodRowsMissing++ >= param.rec.tpc.rebuildTrackExtrConsecGoodRowsMaxGap) { + consecGoodRows = consecGoodRowsMissing = 0; + } + } + } + } + if (lastGoodRow != lastExtrapolateRow) { + for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + candidate.best = -1; + } + } + } + + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); + } + if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { Alpha = prop.GetAlpha(); if (ihitStart != 0) { From ba6593af4e65fd12a40c6ca6a796c8c5e221a59f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 4 Dec 2025 12:05:46 +0100 Subject: [PATCH 26/54] GPU: Add debug option to create temporary MC labels for collected merged tracks --- GPU/GPUTracking/Base/GPUReconstructionCPU.cxx | 4 +-- GPU/GPUTracking/Definitions/GPUSettingsList.h | 13 +++++-- .../GPUTrackingLinkDef_O2_DataTypes.h | 1 + .../Global/GPUChainTrackingMerger.cxx | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 34 ++++++++++++++++++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 13 +++++++ GPU/GPUTracking/qa/GPUQAHelper.h | 11 ++++++ 8 files changed, 76 insertions(+), 6 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx index 9ee6fae1e0fd9..a6d7af2d812aa 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx @@ -216,10 +216,10 @@ int32_t GPUReconstructionCPU::ExitDevice() int32_t GPUReconstructionCPU::RunChains() { mMemoryScalers->temporaryFactor = 1.; - if (GetProcessingSettings().memoryScalingFuzz) { + if (GetProcessingSettings().debug.memoryScalingFuzz) { static std::mt19937 rng; static std::uniform_int_distribution dist(0, 1000000); - uint64_t fuzzFactor = GetProcessingSettings().memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().memoryScalingFuzz; + uint64_t fuzzFactor = GetProcessingSettings().debug.memoryScalingFuzz == 1 ? dist(rng) : GetProcessingSettings().debug.memoryScalingFuzz; GPUInfo("Fuzzing memory scaling factor with %lu", fuzzFactor); mMemoryScalers->fuzzScalingFactor(fuzzFactor); } diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 92e4122fdf649..6221aa5754743 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -32,7 +32,7 @@ BeginNamespace(o2::gpu) // Reconstruction parameters for TPC, no bool in here !!! BeginSubConfig(GPUSettingsRecTPC, tpc, configStandalone.rec, "RECTPC", 0, "Reconstruction settings", rec_tpc) -AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") +AddOptionRTC(rejectQPtB5, float, 1.f / 0.050f, "", 0, "QPt threshold to reject clusters of TPC tracks (Inverse Pt, scaled to B=0.5T!!!)") // TODO: Sort these options automatically for parameter size AddOptionRTC(hitPickUpFactor, float, 1.f, "", 0, "multiplier for the combined cluster+track error during track following") AddOptionRTC(hitSearchArea2, float, 2.f, "", 0, "square of maximum search road of hits during seeding") AddOptionRTC(neighboursSearchArea, float, 3.f, "", 0, "area in cm for the search of neighbours, for z only used if searchWindowDZDR = 0") @@ -330,6 +330,13 @@ AddOption(tpcDecodingSafetyBuffer, uint16_t, 1000, "", 0, "Scaling Factor: safet AddHelp("help", 'h') EndConfig() +// Debug Settings +BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG", 0, "Debugging Settings", proc_debug) +AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) +AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") +AddHelp("help", 'h') +EndConfig() + // Settings steering the processing once the device was selected, only available on the host BeginSubConfig(GPUSettingsProcessing, proc, configStandalone, "PROC", 0, "Processing settings", proc) AddOption(deviceNum, int32_t, -1, "gpuDevice", 0, "Set GPU device to use (-1: automatic, -2: for round-robin usage in timeslice-pipeline)") @@ -359,7 +366,6 @@ AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strateg AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory (0 = disable, 1 / 2 = auto, other values set the size of the memory pool)", min(uint64_t(0))) AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(uint64_t(0))) AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers") -AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations") AddOption(forceMaxMemScalers, uint64_t, 0, "", 0, "Force using the maximum values for all buffers, Set a value n > 1 to rescale all maximums to a memory size of n") AddOption(registerStandaloneInputMemory, bool, false, "registerInputMemory", 0, "Automatically register input memory buffers for the GPU") @@ -406,7 +412,7 @@ AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC de AddOption(tpcApplyCFCutsAtDecoding, bool, false, "", 0, "Apply cluster cuts from clusterization during decoding of compressed clusters") AddOption(tpcApplyClusterFilterOnCPU, uint8_t, 0, "", 0, "Apply custom cluster filter of GPUTPCClusterFilter class, 0: off, 1: debug, 2: PbPb23") AddOption(tpcWriteClustersAfterRejection, bool, false, "", 0, "Apply TPC rejection strategy before writing clusters") -AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") +AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") // TODO: Create some backend-specific options AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from included source code instead of using included spirv code") AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options") AddOption(metalOverrideSourceBuildFlags, std::string, "", "", 0, "Override Metal build flags for compilation from source, put a space for empty options") @@ -430,6 +436,7 @@ AddSubConfig(GPUSettingsProcessingRTCtechnical, rtctech) AddSubConfig(GPUSettingsProcessingParam, param) AddSubConfig(GPUSettingsProcessingNNclusterizer, nn) AddSubConfig(GPUSettingsProcessingScaling, scaling) +AddSubConfig(GPUSettingsProcessingDebug, debug) AddHelp("help", 'h') EndConfig() #endif // !__OPENCL__ && !__METAL__ diff --git a/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h b/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h index 21e495e3ca6c6..2628821bb3ca1 100644 --- a/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h +++ b/GPU/GPUTracking/GPUTrackingLinkDef_O2_DataTypes.h @@ -33,6 +33,7 @@ #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingRTCtechnical + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingNNclusterizer + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingScaling + ; +#pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsProcessingDebug + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplay + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplayInternal + ; #pragma link C++ class o2::gpu::internal::GPUConfigurableParamGPUSettingsDisplayLight + ; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 596bc8f614666..6fe838d939dbb 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -176,6 +176,9 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({{1, -WarpSize(), 0, deviceType}}, 1); runKernel({{1, -WarpSize(), 0, deviceType}}, 1); } + if (!doGPU && GetProcessingSettings().debug.mergerMCLabels) { + Merger.CreateMCLabels(1, 1, 0, 0); + } DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingCollectedTracks, doGPU, Merger, &GPUTPCGMMerger::DumpCollected, *mDebugFile); if (param().rec.tpc.mergeCE) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 5b71478facb0d..53904b8f18c0c 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -148,6 +148,7 @@ using namespace o2::gpu::internal; #include "GPUQA.h" #include "GPUMemorySizeScalers.h" +#include "GPUQAHelper.h" GPUTPCGMMerger::GPUTPCGMMerger() { @@ -164,7 +165,7 @@ GPUTPCGMMerger::GPUTPCGMMerger() } // DEBUG CODE -#if !defined(GPUCA_GPUCODE) && (defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC) +#if defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC #include "GPUQAHelper.h" template @@ -438,6 +439,9 @@ void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) void* GPUTPCGMMerger::SetPointersOutput(void* mem) { computePointerWithAlignment(mem, mMergedTracks, mNMaxTracks); + if (mRec->GetProcessingSettings().debug.mergerMCLabels) { + computePointerWithAlignment(mem, mMergedTrackMC, mNMaxTracks); + } if (mRec->GetParam().dodEdxEnabled) { computePointerWithAlignment(mem, mMergedTracksdEdx, mNMaxTracks); if (mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMask != mRec->GetParam().rec.tpc.dEdxClusterRejectionFlagMaskAlt) { @@ -547,6 +551,34 @@ int32_t GPUTPCGMMerger::CheckSectors() return 0; } +void GPUTPCGMMerger::CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) +{ + const o2::tpc::ClusterNativeAccess* GPUrestrict() clusters = GetConstantMem()->ioPtrs.clustersNative; + if (clusters == nullptr || clusters->clustersMCTruth == nullptr) { + return; + } + if (mMergedTrackMC == nullptr) { + return; + } + + auto labelAssigner = GPUTPCTrkLbl(clusters->clustersMCTruth, 0.1f); + for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { + const auto& trk = mMergedTracks[i]; + if (!trk.OK()) { + continue; + } + labelAssigner.reset(); + for (uint32_t j = 0; j < trk.NClusters(); j++) { + const auto& cl = mClusters[trk.FirstClusterRef() + j]; + if (cl.state & GPUTPCGMMergedTrackHit::flagReject) { + continue; + } + labelAssigner.addLabel(cl.num); + } + mMergedTrackMC[i] = labelAssigner.computeLabel(); + } +} + #endif // GPUCA_GPUCODE GPUd() void GPUTPCGMMerger::ClearTrackLinks(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, bool output) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index ee63269b532fb..774634f34bd1a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -123,6 +123,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() int32_t NMergedTracks() const { return mMemory->nMergedTracks; } GPUhdi() const GPUTPCGMMergedTrack* MergedTracks() const { return mMergedTracks; } + GPUhdi() const o2::MCCompLabel* MergedTrackMC() const { return mMergedTrackMC; } GPUhdi() GPUTPCGMMergedTrack* MergedTracks() { return mMergedTracks; } GPUhdi() const GPUdEdxInfo* MergedTracksdEdx() const { return mMergedTracksdEdx; } GPUhdi() GPUdEdxInfo* MergedTracksdEdx() { return mMergedTracksdEdx; } @@ -214,6 +215,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() void CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE void DumpSectorTracks(std::ostream& out) const; @@ -294,6 +296,7 @@ class GPUTPCGMMerger : public GPUProcessor int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + o2::MCCompLabel* mMergedTrackMC = nullptr; trackCluster* mClusterCandidates = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; int32_t* mHitWeights = nullptr; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index c40236dca1b71..2b37094a53ee4 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -14,6 +14,7 @@ #define GPUCA_CADEBUG 0 #define DEBUG_SINGLE_TRACK -1 +// #define DEBUG_REBUILD_MC #include "GPUTPCDef.h" #include "GPUTPCGMTrackParam.h" @@ -39,6 +40,11 @@ #include "AliHLTTPCClusterMCData.h" #endif +#ifndef GPUCA_GPUCODE +#include "SimulationDataFormat/ConstMCTruthContainer.h" +#include "SimulationDataFormat/MCCompLabel.h" +#endif + #ifndef GPUCA_GPUCODE_DEVICE #include #include @@ -277,6 +283,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; + CADEBUG(merger.MergedTrackMC() printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; @@ -325,6 +332,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; if (candidate.id >= 2) { lastExtrapolateRow = iRow; +#if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) + if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { + int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); + CADEBUG(printf("\t%21sLabel correct: %d\n", "", labelCorrect)); + } +#endif float err2Y, err2Z, xx, yy, zz; const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); diff --git a/GPU/GPUTracking/qa/GPUQAHelper.h b/GPU/GPUTracking/qa/GPUQAHelper.h index dcbe7a94711aa..6712fd0a603ed 100644 --- a/GPU/GPUTracking/qa/GPUQAHelper.h +++ b/GPU/GPUTracking/qa/GPUQAHelper.h @@ -162,6 +162,17 @@ static inline auto GPUTPCTrkLbl(const AliHLTTPCClusterMCLabel* x, Args... args) } } +template +static inline bool GPUTPCTrkLblSearch(const T& clusterLabels, const MCCompLabel& trkLabel) +{ + for (const auto& clLabel : clusterLabels) { + if (trkLabel.compare(clLabel) >= 0) { + return true; + } + } + return false; +} + } // namespace gpu } // namespace o2 From 1166d050f08c4636a45cedf4f6c54fc1f799f89c Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 10 Nov 2026 16:16:25 +0100 Subject: [PATCH 27/54] GPU TPC: Correctly handle rowGap and fit Failures when interpolating / extrapolating --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2b37094a53ee4..480b67d994196 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -108,6 +108,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ N = 0; uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; + bool fitBroken = false; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; @@ -139,11 +140,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); + if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex + rowGap * wayDirection, interpolation.size); + fitBroken = true; + break; + } + uint8_t dEdxSubThresholdRow = 255; - if (lastPropagateRow != 255 && CAMath::Abs(cluster.row - lastPropagateRow) > 1) { + if (rowGap) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); - const uint8_t doInterpolate = (param.rec.tpc.disableRebuildAttachment & 2) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); + const bool crossCE = lastSector != 255 && ((cluster.sector >= GPUTPCGeometry::NSECTORS / 2) != (lastSector >= GPUTPCGeometry::NSECTORS / 2)); + const bool ceOverlap = crossCE && ((cluster.row > lastPropagateRow) != ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1))); // Row gap goes backwards at the CE crossing, these rows were already searched on both sides + const uint8_t doInterpolate = ((param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -158,9 +168,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ break; } FitAddRow(iRow, sector, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, dodEdx, doAttach, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); + if (crossCE && doInterpolate == 2) { // Track crosses the CE within the row gap, search also on the other side, the sector there has the same alpha + FitAddRow(iRow, sector >= GPUTPCGeometry::NSECTORS / 2 ? sector - GPUTPCGeometry::NSECTORS / 2 : sector + GPUTPCGeometry::NSECTORS / 2, iTrk, track, prop, inFlyDirection, merger, &dEdxSubThresholdRow, false, false, doInterpolate, interpolation.hit[index], deltaZ, sumInvSqrtCharge, nAvgCharge); + } } } - interpolationIndex += (CAMath::Abs(cluster.row - lastPropagateRow) - 1) * wayDirection; + interpolationIndex += rowGap * wayDirection; } lastPropagateRow = cluster.row; @@ -187,14 +200,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } + fitBroken = true; break; // bad chi2 for the whole track, stop the fit } - if ((uint32_t)interpolationIndex >= interpolation.size) { - merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); - break; - } - float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { @@ -268,6 +277,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } + fitBroken = true; break; } @@ -280,10 +290,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ deltaZ = 0.f; } - if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && iWay >= nWays - 3 && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; - CADEBUG(merger.MergedTrackMC() printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); + CADEBUG(if (merger.MergedTrackMC()) printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; @@ -320,6 +330,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ sector += 18; } } + if (prop.RotateToAlpha(param.Alpha(sector))) { // GetPropagatedYZ / InverseTransformYZtoX of the next attempt need Y in the frame of the new sector + fail = true; + break; + } } } if (fail) { From bc43f53ffc9336802df6e42135c09f4ae1eb7928 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 10 Nov 2026 20:43:10 +0100 Subject: [PATCH 28/54] GPU TPC: Improve track extrapolation for better efficiency / less fake rate --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 14 ++- .../Global/GPUChainTrackingMerger.cxx | 10 +- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 11 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 106 +++++++++++++----- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 5 files changed, 100 insertions(+), 43 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 6221aa5754743..5f59f4276ddc0 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -104,6 +104,13 @@ AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of sha AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") +AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") +AddOptionRTC(rebuildTrackExtrMaxAvgChi2, float, 4.f, "", 0, "Max average chi2 per cluster to accept an extrapolated segment (0 = disabled)") +AddOptionRTC(rebuildTrackAltCandidateMaxErrFactor, float, 2.f, "", 0, "Alternative cluster candidates of a row are used only if their distance^2 is below this factor times the distance^2 of the best candidate") +AddOptionRTC(rebuildTrackAltCandidateGoodErr, float, 0.25f, "", 0, "Alternative cluster candidates of a row with distance^2 below this value are always used") +AddOptionRTC(rebuildTrackErrFactorOneSided, float, 1.5f, "", 0, "Factor applied to the error (distance^2) of cluster candidates found with the track position from only one side during rebuild") +AddOptionRTC(rebuildTrackErrFactorExtrapolation, float, 1.f, "", 0, "Factor applied to the error (distance^2) of cluster candidates found by track extrapolation during rebuild") +AddOptionRTC(rebuildTrackMaxInterpolationDistY, float, 4.f, "", 0, "Max distance in Y between the track positions from both sides to use their interpolation for the cluster search in rows without cluster during rebuild") AddOptionRTC(maxTimeBinAboveThresholdIn1000Bin, uint16_t, 500, "", 0, "Except pad from cluster finding if total number of charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(maxConsecTimeBinAboveThreshold, uint16_t, 200, "", 0, "Except pad from cluster finding if number of consecutive charges in a fragment is above this baseline (disable = 0)") AddOptionRTC(noisyPadSaturationThreshold, uint16_t, 700, "", 0, "Threshold where a timebin is considered saturated, disabling the noisy pad check for that pad") @@ -168,9 +175,12 @@ AddOptionRTC(rejectEdgeClustersInSeeding, int8_t, 0, "", 0, "Reject edge cluster AddOptionRTC(rejectEdgeClustersInTrackFit, int8_t, 0, "", 0, "Reject edge clusters based on uncorrected track Y during track fit") AddOptionRTC(tubeExtraProtectMinRow, uint8_t, 20, "", 0, "Increase Protection, decrease removal by factor 2, when below this row") AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, decrease removal by factor 2, when on this number of pads from the edge") -AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 15, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") -AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 6, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 20, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 8, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrMinConsecGoodRowsEdge, uint8_t, 3, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") +AddOptionRTC(rebuildTrackCandidateErrChi2, uint8_t, 0, "", 0, "Rank cluster candidates during rebuild by chi2 instead of distance^2 (rebuildTrackAltCandidateGoodErr is then in units of chi2)") +AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 6fe838d939dbb..9156b7700af9f 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -263,10 +263,12 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel(GetGridAuto(0, deviceType)); if (param().rec.tpc.mergeLoopersAfterburner) { runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0, deviceType) : GetGridAuto(0, deviceType)); - if (doGPU) { - TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory(), 0); - SynchronizeStream(0); // TODO: could probably synchronize on an event after runKernel - } + } + if (doGPU) { + TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResMemory(), 0); + SynchronizeStream(0); // TODO: could probably synchronize on an event after runKernel + } + if (param().rec.tpc.mergeLoopersAfterburner) { runKernel(GetGridAuto(0, deviceType)); runKernel(doGPU ? GetGrid(Merger.Memory()->nLooperMatchCandidates, 0, deviceType) : GetGridAuto(0, deviceType)); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingLoopers, Merger, &GPUTPCGMMerger::DumpLoopers, *mDebugFile); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 53904b8f18c0c..c477847213003 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2184,7 +2184,7 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { candidates[0].id = cl.num + 2; - candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly + candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly. Tracks with NDF <= 0 get no fallback in rows with candidates, and get those back only via the shared recovery candidates[0].weight = cl.state; candidates[0].sector = cl.sector; lastRow = cl.row; @@ -2204,15 +2204,14 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (candidates[0].best == 0) { - if (candidates[iteration].id >= 2) { - const int32_t nCl = trk.NClusters(); - float weightf = nCl * (nCl * 2 - 5) * 128 / trk.GetParam().GetChi2() / (1.f + candidates[iteration].error); + if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster + float weightf = (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); candidates[iteration].weight = weight; CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); } else { - candidates[0].best = -1; + candidates[0].best = -2; } } } @@ -2259,7 +2258,7 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads if (candidates[0].best != 128 && candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations } - } else if (candidates[0].best == 0 && candidates[0].id >= 2) { + } else if (candidates[0].best != -1 && candidates[0].id >= 2) { sharedRowCandidates[nSharedRowCandidates++] = j; } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 480b67d994196..97ba5b6aa964f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -83,6 +83,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; + // TODO: Could do 5 ways and interpolate again on the rebuilt clusters and reject in the final way based on that (and get the outer param from the rebuilt clusters), but costs 2 more ways for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! @@ -188,7 +189,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ nMissed2++; NTolerated++; if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); // TODO: mark clusters from single-sided finding as dubious, and apply stricter restriction cut later + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); } continue; } @@ -290,23 +291,25 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ deltaZ = 0.f; } - if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255) { + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255 && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; CADEBUG(if (merger.MergedTrackMC()) printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); uint8_t rowGapActive = 0, rowGapTotal = 0, missingRowsTotal = 0; - uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow; - uint8_t consecGoodRows = param.rec.tpc.rebuildTrackExtrMinConsecGoodRows, consecGoodRowsMissing = 0; + uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow, lastFoundRow = lastPropagateRow; + uint8_t consecGoodRows = ihitStart == ((iWay & 1) ? 0 : maxN - 1) ? param.rec.tpc.rebuildTrackExtrMinConsecGoodRows : 0, consecGoodRowsMissing = 0, segmentRows = 0; // Continue the track directly only if its last hit was accepted in the fit + float segmentStartChi2 = mChi2; prop.SetTrack(this, prop.GetAlpha()); - for (uint32_t iRow = lastPropagateRow + up; iRow < GPUTPCGeometry::NROWS; iRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers + uint32_t iExtrapolatedRow = lastPropagateRow + up; + for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers bool fail = false; for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { float tmpX, tmpY, tmpZ; - if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { + if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iExtrapolatedRow), tmpY, tmpZ)) { fail = true; break; } - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iRow, tmpY, tmpZ, tmpX); + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iExtrapolatedRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { fail = true; break; @@ -339,13 +342,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (fail) { break; } - CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); gputpcgmmergertypes::InterpolationErrorHit inter; inter.markInvalid(); - float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - if (candidate.id >= 2) { - lastExtrapolateRow = iRow; + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, true); + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; + bool found = candidate.id >= 2; + if (found) { + lastExtrapolateRow = iExtrapolatedRow; #if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); @@ -354,7 +358,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ #endif float err2Y, err2Z, xx, yy, zz; const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; - merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); + merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { candidate.best = -1; break; @@ -365,16 +369,30 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float invAvgCharge = (sumInvSqrtCharge += invSqrtCharge) / ++nAvgCharge; invAvgCharge *= invAvgCharge; const uint8_t clusterState = cl.getFlags(); - prop.GetErr2(err2Y, err2Z, param, zz, iRow, clusterState, sector, time, invAvgCharge, invCharge); + prop.GetErr2(err2Y, err2Z, param, zz, iExtrapolatedRow, clusterState, sector, time, invAvgCharge, invCharge); CADEBUG(printf("\t%21sResiduals %8.3f %8.3f --- Errors %8.3f %8.3f\n", "", yy - mP[0], zz - mP[1], CAMath::Sqrt(err2Y), CAMath::Sqrt(err2Z))); - uint32_t retValUpd = prop.Update(yy, zz, iRow, param, clusterState, true, refit, err2Y, err2Z); - CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (consecGoodRows == 0) { // New segment + segmentStartChi2 = mChi2; + segmentRows = 0; + } + int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); + CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd != 0) { candidate.best = -1; - break; + if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { + break; + } + found = false; // Cluster rejected by chi2, handle as missing row + } else { + segmentRows++; } + } + if (found) { + lastFoundRow = iExtrapolatedRow; if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { - lastGoodRow = iRow; + if (param.rec.tpc.rebuildTrackExtrMaxAvgChi2 <= 0 || mChi2 - segmentStartChi2 <= param.rec.tpc.rebuildTrackExtrMaxAvgChi2 * segmentRows) { + lastGoodRow = iExtrapolatedRow; // TODO: Could optionally store the track state here, and revert to the good state in case we reject further clusters + } consecGoodRowsMissing = 0; } rowGapActive = rowGapTotal = 0; @@ -385,8 +403,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (++rowGapTotal > param.rec.tpc.trackFollowingMaxRowGap) { consecGoodRows = consecGoodRowsMissing = 0; } - uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iRow, uncorrectedY)); - if (pad < GPUTPCGeometry::NPads(iRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iRow, pad))) { + uint32_t pad = CAMath::Float2UIntRn(GPUTPCGeometry::LinearY2Pad(sector, iExtrapolatedRow, uncorrectedY)); + if (pad < GPUTPCGeometry::NPads(iExtrapolatedRow) && (!merger.GetConstantMem()->calibObjects.dEdxCalibContainer || !merger.GetConstantMem()->calibObjects.dEdxCalibContainer->isDead(sector, iExtrapolatedRow, pad))) { if (rowGapActive++ >= param.rec.tpc.trackFollowingMaxRowGap) { break; } @@ -396,6 +414,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } + if (iExtrapolatedRow >= GPUTPCGeometry::NROWS && consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRowsEdge && (param.rec.tpc.rebuildTrackExtrMaxAvgChi2 <= 0 || mChi2 - segmentStartChi2 <= param.rec.tpc.rebuildTrackExtrMaxAvgChi2 * segmentRows)) { // Reached the inner / outer end of the TPC, a shorter last segment is sufficient + lastGoodRow = lastFoundRow; + } if (lastGoodRow != lastExtrapolateRow) { for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; @@ -464,7 +485,7 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } if (doInterpolate) { if (doInterpolate == 2) { - if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > 4.f) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { inter.markInvalid(); } FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); @@ -572,7 +593,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly) +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, bool extrapolation) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -580,7 +601,13 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre GPUglobalref() const cahit2* hits = tracker.HitData(rowData); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(rowData); float uncorrectedY = -1e6f, uncorrectedZ; - if (rowData.NHits() && (inter.isValid() || (!interOnly && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && mC[0] < param.rec.tpc.rebuildTrackMaxNonIntCov && mC[2] < param.rec.tpc.rebuildTrackMaxNonIntCov))) { + const float maxNonIntCov = param.rec.tpc.rebuildTrackMaxNonIntCov; + bool useInter = inter.isValid(); + if (useInter && interOnly) { // One-sided from the other direction, same limit as for the current track position + useInter = (float)inter.errorY < maxNonIntCov && (float)inter.errorZ < maxNonIntCov; + } + const bool useCurrent = !interOnly && maxNonIntCov > 0 && mC[0] < maxNonIntCov && mC[2] < maxNonIntCov && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF; + if (rowData.NHits() && (useInter || useCurrent)) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -594,7 +621,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre ImP1 = (float)inter.posZ + deltaZ; ImC0 = (float)inter.errorY; ImC2 = (float)inter.errorZ; - } else if (inter.isValid()) { + } else if (useInter) { const float Iz0 = (float)inter.posY - mP[0]; const float Iz1 = (float)inter.posZ + deltaZ - mP[1]; const float Iw0 = 1.f / (mC[0] + (float)inter.errorY); @@ -614,21 +641,36 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + float xMoved = mX, yMoved = mP[0], zMoved = mP[1], sinPhiMoved = prop.GetSinPhi0(); + if (interOnly) { // Track is not at this row: estimate track position and angle there (for the cluster errors) from the interpolated position and the helix of the current linearization + float sA, cA; + CAMath::SinCos(param.Alpha(sector) - prop.GetAlpha(), sA, cA); + xMoved = GPUTPCGeometry::Row2X(row); + yMoved = ImP0; + zMoved = ImP1; + const float sinPhi = prop.GetSinPhi0() - prop.GetQPt0() * prop.GetBz(mX, mP[0], mP[1]) * (xMoved * cA - yMoved * sA - mX); // at the X of the row in the current track frame + if (CAMath::Abs(sinPhi) < constants::MAX_SIN_PHI) { + sinPhiMoved = sinPhi * cA - CAMath::Copysign(CAMath::Sqrt(1.f - sinPhi * sinPhi), prop.GetCosPhi0()) * sA; + } + sinPhiMoved = CAMath::Clamp(sinPhiMoved, -constants::MAX_SIN_PHI, constants::MAX_SIN_PHI); + } + int32_t nCandidates = 0; while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { float err2Y, err2Z; - param.GetClusterErrors2(sector, row, mP[1], mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + param.GetClusterErrors2(sector, row, zMoved, sinPhiMoved, mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge const float kFactor = tracker.GetChiSeedFactor(); - const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(mC[0]))); // TODO: is 4 a good factor?? - const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(mC[2]))); + const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(ImC0))); // TODO: is 4 a good factor?? + const float sz2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Z + CAMath::Abs(ImC2))); const float tubeY = CAMath::Sqrt(sy2); const float tubeZ = CAMath::Sqrt(sz2); rowData.Grid().GetBinArea(uncorrectedY, uncorrectedZ + zOffset, tubeY, tubeZ, bin, ny, nz); + const float errFactor = extrapolation ? param.rec.tpc.rebuildTrackErrFactorExtrapolation : ((interOnly || !useInter) ? param.rec.tpc.rebuildTrackErrFactorOneSided : 1.f); // Downweight candidates not found by two-sided interpolation const int32_t nBinsY = rowData.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[rowData.HitNumberOffset()]); @@ -653,13 +695,17 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre float invAvgCharge = (sumInvSqrtCharge + invSqrtCharge) / (nAvgCharge + 1); invAvgCharge *= invAvgCharge; - prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + if (interOnly) { + GPUTPCGMPropagator::GetErr2(err2YA, err2ZA, param, sinPhiMoved, mP[3], zMoved, xMoved, yMoved, row, clflags, sector, time, invAvgCharge, invCharge, false); + } else { + prop.GetErr2(err2YA, err2ZA, param, mP[1], row, clflags, sector, time, invAvgCharge, invCharge); + } const float Jw0 = 1.f / (ImC0 + err2YA); const float Jw2 = 1.f / (ImC2 + err2ZA); const float chi2Y = Jw0 * dy * dy; const float chi2Z = Jw2 * dz * dz; bool ok = !prop.RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clflags); - float err = dy * dy + dz * dz; + const float err = (param.rec.tpc.rebuildTrackCandidateErrChi2 ? (chi2Y + chi2Z) : (dy * dy + dz * dz)) * errFactor; if (ok) { int32_t insert = nCandidates; for (int32_t c = 0; c < nCandidates; c++) { @@ -1304,7 +1350,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() } CADEBUG(printf("Track rejected, marking for retry\n")); uint32_t nRefit = CAMath::AtomicAdd(&merger.Memory()->nRetryRefit, 1u); - merger.RetryRefitIds()[nRefit] = iTrk; + merger.RetryRefitIds()[nRefit] = iTrk; // TODO: With rebuildTrackInFit the retry runs on the rebuilt clusters, so it cannot recover tracks that lost their clusters in the rebuild return; } if (retryAttempt && (t.mNDF < 10 || t.mChi2 / t.mNDF >= 6.f)) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 6b5322a8ec1cf..4619db960981d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,7 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly); + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, bool extrapolation = false); GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use From 7d4d7c63ec845eb8ee4371af1223e0c9f136f363 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 11 Nov 2026 13:58:59 +0100 Subject: [PATCH 29/54] GPU TPC: Let fallback clusters compete for ambiguity solving as all other clusters --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 ++ GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 5f59f4276ddc0..627857d8a9530 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -106,6 +106,8 @@ AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") AddOptionRTC(rebuildTrackExtrMaxAvgChi2, float, 4.f, "", 0, "Max average chi2 per cluster to accept an extrapolated segment (0 = disabled)") +AddOptionRTC(rebuildTrackFallbackErr, float, 1.f, "", 0, "Error (distance^2) assigned to the original clusters of a track used as fallback candidates in rows that could not be searched during rebuild") +AddOptionRTC(rebuildTrackFallbackTrackChi2, float, 4.f, "", 0, "Chi2 per cluster assumed for the hit weight of tracks without successful fit (NDF <= 0) during rebuild") AddOptionRTC(rebuildTrackAltCandidateMaxErrFactor, float, 2.f, "", 0, "Alternative cluster candidates of a row are used only if their distance^2 is below this factor times the distance^2 of the best candidate") AddOptionRTC(rebuildTrackAltCandidateGoodErr, float, 0.25f, "", 0, "Alternative cluster candidates of a row with distance^2 below this value are always used") AddOptionRTC(rebuildTrackErrFactorOneSided, float, 1.5f, "", 0, "Factor applied to the error (distance^2) of cluster candidates found with the track position from only one side during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index c477847213003..5b40560d9b61a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -2178,15 +2178,18 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, } mTrackRebuildHelper[i].reverse = mClusters[trk.FirstClusterRef()].row < mClusters[trk.FirstClusterRef() + trk.NClusters() - 1].row; if (!(Param().rec.tpc.disableRebuildAttachment & 8)) { + const uint32_t nMaxCandidates = Param().rec.tpc.rebuildTrackInFitClusterCandidates; int lastRow = -1; for (uint32_t j = 0; j < trk.NClusters(); j++) { const auto& cl = mClusters[trk.FirstClusterRef() + j]; - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (cl.row != lastRow && candidates[0].id == 0 && (!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0)) { - candidates[0].id = cl.num + 2; - candidates[0].best = 128; // TODO: Needs better handling in the future, particularly should be marked as shared correctly. Tracks with NDF <= 0 get no fallback in rows with candidates, and get those back only via the shared recovery - candidates[0].weight = cl.state; - candidates[0].sector = cl.sector; + auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * nMaxCandidates]; + if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].id == 0 || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error + for (uint32_t k = 0; k < nMaxCandidates; k++) { + if (candidates[k].id == 0) { + candidates[k] = {.id = cl.num + 2, .row = cl.row, .sector = cl.sector, .error = Param().rec.tpc.rebuildTrackFallbackErr, .weight = 0, .best = 0}; + break; + } + } lastRow = cl.row; } } @@ -2198,15 +2201,15 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, { for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { const auto& trk = mMergedTracks[i]; - if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + if (!trk.OK()) { continue; } for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (candidates[0].best == 0) { - if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster - float weightf = (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); - int32_t weight = weightf < 0.f ? 0 : (weightf > 2e9f ? 2e9f : (int32_t)weightf); + if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster + float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2) / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); // Tracks with NDF <= 0 use their nominal number of clusters and assume a fixed chi2 per cluster + int32_t weight = weightf > 0.f ? (weightf > 2e9f ? 2e9f : (int32_t)weightf) : 0; candidates[iteration].weight = weight; CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); @@ -2222,7 +2225,7 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads { for (int32_t i = get_global_id(0); i < NMergedTracks(); i += get_global_size(0)) { auto& trk = mMergedTracks[i]; - if (!trk.OK() || trk.GetParam().GetNDF() <= 0) { + if (!trk.OK()) { continue; } for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { @@ -2250,12 +2253,12 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads uint8_t nSharedRowCandidates = 0; for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (trk.GetParam().GetNDF() > 0 && candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { + if (candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { candidates[0].best = nIter; } if (candidates[0].best > 0) { nHits++; - if (candidates[0].best != 128 && candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { + if (candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations } } else if (candidates[0].best != -1 && candidates[0].id >= 2) { @@ -2300,11 +2303,6 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; if (candidates[0].best > 0) { auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; - if (candidates[0].best == 128) { - outCl = {.num = candidates[0].id - 2, .sector = candidates[0].sector, .row = (uint8_t)j, .state = (uint8_t)candidates[0].weight}; - written++; - continue; - } const auto& best = candidates[candidates[0].best - 1]; const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); From 6158e28131c8ec1662f1811fd0ca55b43928f3ec Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 11 Nov 2026 18:55:59 +0100 Subject: [PATCH 30/54] GPU TPC: Add debug option to dump merged track ncl to text file --- GPU/GPUTracking/Base/GPUReconstruction.cxx | 2 +- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.h | 3 ++ .../GPUChainTrackingDebugAndProfiling.cxx | 23 +++++++++++ .../Global/GPUChainTrackingMerger.cxx | 3 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 5 ++- .../Standalone/Benchmark/CMakeLists.txt | 1 + .../Standalone/scripts/compareTrackNcl.py | 40 +++++++++++++++++++ 8 files changed, 76 insertions(+), 2 deletions(-) create mode 100755 GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx index 3e7509d8fad73..c1a09f4075bc9 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.cxx +++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx @@ -295,7 +295,7 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice() if (GetProcessingSettings().deterministicGPUReconstruction && GetProcessingSettings().debugLevel >= 6) { mProcessingSettings->nTPCClustererLanes = 1; } - if (GetProcessingSettings().createO2Output > 1 && GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) { + if (GetProcessingSettings().createO2Output > 1 && ((GetProcessingSettings().runQA && GetProcessingSettings().qcRunFraction == 100.f) || GetProcessingSettings().debug.mergerDumpTrackNCl)) { mProcessingSettings->createO2Output = 1; } if (!GetProcessingSettings().createO2Output || !IsGPU()) { diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 627857d8a9530..0cbd4955c4b64 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -346,6 +346,7 @@ EndConfig() BeginSubConfig(GPUSettingsProcessingDebug, debug, configStandalone.proc, "DEBUG", 0, "Debugging Settings", proc_debug) AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1)) AddOption(mergerMCLabels, bool, false, "", 0, "Create MC labels for merged tracks before refit for debugging") +AddOption(mergerDumpTrackNCl, bool, false, "", 0, "Dump file with nCl per merged track fit") AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 1507147bbafda..92eda50194c74 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -322,6 +322,9 @@ class GPUChainTracking : public GPUChain int32_t RunTRDTrackingInternal(); uint32_t StreamForSector(uint32_t sector) const; + // Debug + void DoDebugMergerNClDump(const GPUTPCGMMerger& merger); + std::mutex mMutexUpdateCalib; std::unique_ptr mPipelineFinalizationCtx; GPUChainTrackingFinalContext* mPipelineNotifyCtx = nullptr; diff --git a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx index e45aeb4ba9246..13369405ac519 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingDebugAndProfiling.cxx @@ -419,3 +419,26 @@ void GPUChainTracking::DoDebugRawDump() mRec->DumpSettings((dirName + "/").c_str()); DumpData((dirName + "/event.0.dump").c_str(), &ioPtrs); } + +void GPUChainTracking::DoDebugMergerNClDump(const GPUTPCGMMerger& merger) +{ + if (GetProcessingSettings().debug.mergerDumpTrackNCl) { +#ifndef GPUCA_DETERMINISTIC_MODE + GPUError("GPUCA_DETERMINISTIC_MODE not enabled, track dump will not be deterministic"); +#endif + if (!GetProcessingSettings().deterministicGPUReconstruction) { + GPUError("deterministicGPUReconstruction not enabled, track dump will not be deterministic"); + } + if (GetRecoStepsGPU() & RecoStep::TPCMerging) { + SynchronizeGPU(); + } + FILE* fout = fopen("merger_tracks_ncl.txt", "w+"); + if (fout == nullptr) { + throw std::runtime_error("Error Opening Debug File"); + } + for (int32_t i = 0; i < merger.NMergedTracks(); i++) { + fprintf(fout, "Track %7d NCl %5u\n", i, merger.MergedTracks()[i].OK() ? merger.MergedTracks()[i].NClustersFitted() : 0u); + } + fclose(fout); + } +} diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 9156b7700af9f..6786f7fc927e9 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -235,6 +235,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (param().rec.tpc.rebuildTrackInFit) { runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); } + // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); if (param().rec.tpc.rebuildTrackInFit) { runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); @@ -306,6 +307,8 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) if (GetProcessingSettings().keepDisplayMemory && !GetProcessingSettings().keepAllMemory) { TransferMemoryResourcesToHost(RecoStep::TPCMerging, &Merger, -1, true); } + DoDebugMergerNClDump(Merger); + // mRec->SetNActiveThreads(-1); mRec->ReturnVolatileDeviceMemory(); mRec->PopNonPersistentMemory(RecoStep::TPCMerging, qStr2Tag("TPCMERGE")); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 97ba5b6aa964f..c3d254a0b9cc2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -37,6 +37,9 @@ #ifdef GPUCA_CADEBUG_ENABLED #include "GPUSettings.h" +#include +#include +#include #include "AliHLTTPCClusterMCData.h" #endif @@ -1335,7 +1338,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - CADEBUG(if (DEBUG_SINGLE_TRACK != -1 && iTrk != ((DEBUG_SINGLE_TRACK == -2 && getenv("DEBUG_TRACK")) ? atoi(getenv("DEBUG_TRACK")) : DEBUG_SINGLE_TRACK)) { track.SetNClusters(0); track.SetOK(0); return; }); + CADEBUG(if (DEBUG_SINGLE_TRACK != -1) { static const std::set dbgTracks = []() { std::set m; if (const char* e = DEBUG_SINGLE_TRACK == -2 ? getenv("DEBUG_TRACK") : nullptr) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } else { m.insert(DEBUG_SINGLE_TRACK); } return m; }(); if (!dbgTracks.count(iTrk)) { track.SetNClusters(0); track.SetOK(0); return; } }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt b/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt index d3bef7c0be1f6..efd165047fb7f 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt +++ b/GPU/GPUTracking/Standalone/Benchmark/CMakeLists.txt @@ -32,6 +32,7 @@ if(ALIGPU_BUILD_TYPE STREQUAL "Standalone") endif() install(DIRECTORY ../tools DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU) +install(DIRECTORY ../scripts DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU USE_SOURCE_PERMISSIONS) install(DIRECTORY ../../Definitions/Parameters/ DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/GPU/tools FILES_MATCHING REGEX "\\.(python|sh|cmake)") target_compile_definitions(${targetName} PRIVATE $) diff --git a/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py new file mode 100755 index 0000000000000..01ff969cf17a6 --- /dev/null +++ b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# Copyright 2019-2026 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +# Usage: compareTrackNcl.py FILE1 FILE2, with files of lines "Track NCl " (from --DEBUGmergerDumpTrackNCl). +# Prints the ids of tracks with less than half the clusters in FILE1 compared to FILE2 as DEBUG_TRACK="...", to be used as environment variable with DEBUG_SINGLE_TRACK -2 in GPUTPCGMTrackParam.cxx, followed by some statistics. + +import sys + + +def readNCl(fileName): + ncl = {} + with open(fileName) as f: + for line in f: + cols = line.split() + if len(cols) >= 4 and cols[0] == "Track" and cols[2] == "NCl": + ncl[int(cols[1])] = int(cols[3]) + return ncl + + +if len(sys.argv) != 3: + sys.exit("Usage: " + sys.argv[0] + " FILE1 FILE2") +ncl1, ncl2 = readNCl(sys.argv[1]), readNCl(sys.argv[2]) +common = [i for i in sorted(ncl1) if i in ncl2] +ids = [str(i) for i in common if 2 * ncl1[i] < ncl2[i]] +print('DEBUG_TRACK="' + "\n".join(",".join(ids[i:i + 30]) for i in range(0, len(ids), 30)) + '"') +print() # Statistics as shell comments, so that the output can still be passed to eval +print("# Tracks in total (present in both files):", len(common)) +for text, n in (("Tracks with 0 clusters in file 2", sum(1 for i in common if ncl2[i] == 0)), + ("Tracks with 0 clusters in file 1 and > 0 in file 2", sum(1 for i in common if ncl1[i] == 0 and ncl2[i] > 0)), + ("Tracks with NCl(file 1) < 0.5 * NCl(file 2)", len(ids))): + print("# %s: %d (%.2f%%)" % (text, n, 100. * n / max(1, len(common)))) From 73793ea175be4c241270c0c61de46c8550b3eb36 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 12:38:53 +0100 Subject: [PATCH 31/54] GPU TPC: Continuous and less strict (during refit) NCl cut for given Pt --- GPU/GPUTracking/Base/GPUParam.h | 4 ++-- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 7 ++++++- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx | 2 +- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 2 +- .../SectorTracker/GPUTPCTrackletConstructor.cxx | 4 ++-- GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx | 2 +- 8 files changed, 15 insertions(+), 8 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUParam.h b/GPU/GPUTracking/Base/GPUParam.h index 30cc4d1e7dca9..5fab4b5e0fb54 100644 --- a/GPU/GPUTracking/Base/GPUParam.h +++ b/GPU/GPUTracking/Base/GPUParam.h @@ -80,8 +80,8 @@ struct GPUParam : public internal::GPUParam_t void UpdateRun3ClusterErrors(const float* yErrorParam, const float* zErrorParam); #endif - GPUd() constexpr uint32_t tpcMinHitsB5(float qPtB5) const { return CAMath::Abs(qPtB5) > 10 ? 10 : (CAMath::Abs(qPtB5) > 5 ? 15 : 29); } // Minimum hits should depend on Pt, low Pt tracks can have few hits. 29 Hits default, 15 for < 200 mev, 10 for < 100 mev - GPUd() constexpr float Alpha(int32_t iSector) const // TODO: Check if this is faster, or the lookup from GPUTPCGeometry + GPUd() constexpr uint32_t tpcMinHitsB5Abs(float qPtB5) const { return qPtB5 > 10.f ? 10 : (qPtB5 > 5.f ? (uint32_t)(20 - qPtB5) : (qPtB5 > 3.6f ? (uint32_t)(65 - 10.f * qPtB5) : 29)); } + GPUd() constexpr float Alpha(int32_t iSector) const // TODO: Check if this is faster, or the lookup from GPUTPCGeometry { if (iSector >= (int32_t)o2::tpc::constants::MAXSECTOR / 2) { iSector -= o2::tpc::constants::MAXSECTOR / 2; diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 0cbd4955c4b64..fbedbd854833b 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -102,6 +102,7 @@ AddOptionRTC(rejectEdgeClustersMargin, float, 0.f, "", 0, "Margin in cm of Y pos AddOptionRTC(rejectEdgeClustersSigmaMargin, float, 0.f, "", 0, "Margin factor for trackSigmaY when rejecting edge clusters based on uncorrected track Y") AddOptionRTC(trackletMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for tracklet") AddOptionRTC(trackletMinSharedNormFactor, float, 0.f, "", 0, "Max shared defined as trackletMinSharedNormFactor*max(current_nhits,trackletMinSharedNormFactor*minHits,1)") +AddOptionRTC(trackFitMinHitsMaxQPtIncrease, float, 0.2f, "", 0, "For the QPt-dependent minimum number of hits of the track fit, abs(QPt) is increased by 1 sigma, but at most by this fraction (0 = use fitted QPt)") AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 5b40560d9b61a..26ffff549bad2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1701,7 +1701,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread } nHits += nTrackHits; } - if (nHits < Param().tpcMinHitsB5(trbase->QPt() * Param().qptB5Scaler)) { + if (nHits < Param().tpcMinHitsB5Abs(CAMath::Abs(trbase->QPt() * Param().qptB5Scaler))) { break; } @@ -2163,6 +2163,11 @@ GPUd() void GPUTPCGMMerger::MergeLoopersMain(int32_t nBlocks, int32_t nThreads, #endif*/ } +GPUd() uint32_t GPUTPCGMMerger::MinHitsForQPt(float qPt, float err2QPt) const +{ + return Param().tpcMinHitsB5Abs((CAMath::Abs(qPt) + CAMath::Min(CAMath::Sqrt(CAMath::Max(0.f, err2QPt)), Param().rec.tpc.trackFitMinHitsMaxQPtIncrease * CAMath::Abs(qPt))) * Param().qptB5Scaler); +} + GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 774634f34bd1a..1ca312165f17a 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -215,6 +215,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights2(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ResolveHitWeightsShared(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() uint32_t MinHitsForQPt(float qPt, float err2QPt) const; // QPt-dependent minimum number of hits, for QPt increased by 1 sigma (at most by the fraction trackFitMinHitsMaxQPtIncrease) GPUd() void CreateMCLabels(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); #ifndef GPUCA_GPUCODE diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index 4bcffd8a31c46..ddd6862b96bbb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -76,7 +76,7 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlock if (nCl == 0) { continue; } - if (nCl + 2 < merger.Param().tpcMinHitsB5(tracks[i].GetParam().GetQPt() * merger.Param().qptB5Scaler)) { // Give 2 hits tolerance in the primary leg, compared to the full fit of the looper + if (nCl + 2 < merger.MinHitsForQPt(tracks[i].GetParam().GetQPt(), tracks[i].GetParam().GetErr2QPt())) { // Give 2 hits tolerance in the primary leg, compared to the full fit of the looper continue; } if (merger.Param().rec.tpc.minNClustersFinalTrack != -1 && nCl < (uint32_t)merger.Param().rec.tpc.minNClustersFinalTrack) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index c3d254a0b9cc2..a8a71f124ab49 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -447,7 +447,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ o2::utils::DebugStreamer::instance()->getStreamer("debug_accept_track", "UPDATE") << o2::utils::DebugStreamer::instance()->getUniqueTreeName("debug_accept_track").data() << "iTrk=" << iTrk << "outerParam=" << track.OuterParam() << "track=" << this << "ihitStart=" << ihitStart << "\n"; }) - if (!(N + NTolerated >= (int32_t)param.tpcMinHitsB5(mP[4] * param.qptB5Scaler) && 2 * NTolerated <= CAMath::Max(10, N) && CheckNumericalQuality(covYYUpd))) { + if (!(N + NTolerated >= (int32_t)merger.MinHitsForQPt(mP[4], mC[14]) && 2 * NTolerated <= CAMath::Max(10, N) && CheckNumericalQuality(covYYUpd))) { return false; // TODO: NTolerated should never become that large, check what is going wrong! } if (param.rec.tpc.minNClustersFinalTrack != -1 && N + NTolerated < param.rec.tpc.minNClustersFinalTrack) { diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx index 86979c29006ae..2f89a998f8e9f 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx @@ -56,8 +56,8 @@ GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_ { // reconstruction of tracklets, tracklet store step const uint32_t nHits = r.mLastRow + 1 - r.mFirstRow; - if (nHits == 0 || r.mNHits == 0 || (r.mNHits < tracker.Param().tpcMinHitsB5(tParam.QPt() * tracker.Param().qptB5Scaler) || !CheckCov(tParam) || CAMath::Abs(tParam.GetQPt() * tracker.Param().qptB5Scaler) > tracker.Param().rec.maxTrackQPtB5)) { - CADEBUG(printf(" Rejected: nHits %d QPt %f MinHits %d MaxQPt %f CheckCov %d\n", r.mNHits, tParam.QPt(), tracker.Param().tpcMinHitsB5(tParam.QPt() * tracker.Param().qptB5Scaler), tracker.Param().rec.maxTrackQPtB5, (int32_t)CheckCov(tParam))); + if (nHits == 0 || r.mNHits == 0 || (r.mNHits < tracker.Param().tpcMinHitsB5Abs(CAMath::Abs(tParam.QPt() * tracker.Param().qptB5Scaler)) || !CheckCov(tParam) || CAMath::Abs(tParam.GetQPt() * tracker.Param().qptB5Scaler) > tracker.Param().rec.maxTrackQPtB5)) { + CADEBUG(printf(" Rejected: nHits %d QPt %f MinHits %d MaxQPt %f CheckCov %d\n", r.mNHits, tParam.QPt(), tracker.Param().tpcMinHitsB5Abs(CAMath::Abs(tParam.QPt() * tracker.Param().qptB5Scaler)), tracker.Param().rec.maxTrackQPtB5, (int32_t)CheckCov(tParam))); return; } diff --git a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx index 1f85718de6fa5..0a68a64c9a204 100644 --- a/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx +++ b/GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx @@ -51,7 +51,7 @@ GPUdii() void GPUTPCTrackletSelector::Thread<0>(int32_t nBlocks, int32_t nThread uint32_t gap = 0; uint32_t nShared = 0; uint32_t nHits = 0; - const uint32_t minHits = tracker.Param().rec.tpc.minNClustersTrackSeed == -1 ? tracker.Param().tpcMinHitsB5(tracklet.Param().QPt() * tracker.Param().qptB5Scaler) : tracker.Param().rec.tpc.minNClustersTrackSeed; + const uint32_t minHits = tracker.Param().rec.tpc.minNClustersTrackSeed == -1 ? tracker.Param().tpcMinHitsB5Abs(CAMath::Abs(tracklet.Param().QPt() * tracker.Param().qptB5Scaler)) : tracker.Param().rec.tpc.minNClustersTrackSeed; const uint32_t sharingMinNorm = minHits * tracker.Param().rec.tpc.trackletMinSharedNormFactor; const float maxSharedNorm = maxSharedFrac * sharingMinNorm; From 9a9226b902a4b1e277d5df2cfcd1cb5d691737cd Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 13:16:41 +0100 Subject: [PATCH 32/54] GPU GPUCA_CADEBUG: add possibility to have a condition for the debug output --- GPU/GPUTracking/Definitions/GPUDef.h | 22 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 6 ++--- .../Standalone/scripts/compareTrackNcl.py | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUDef.h b/GPU/GPUTracking/Definitions/GPUDef.h index 54c180d5f7187..d3aa2acfa51d5 100644 --- a/GPU/GPUTracking/Definitions/GPUDef.h +++ b/GPU/GPUTracking/Definitions/GPUDef.h @@ -70,10 +70,23 @@ #ifdef CADEBUG #undef CADEBUG #endif - #if GPUCA_CADEBUG == 1 && !defined(GPUCA_GPUCODE) - #define CADEBUG(...) __VA_ARGS__ - #define CADEBUG2(cmd, ...) {__VA_ARGS__; cmd;} - #define GPUCA_CADEBUG_ENABLED + #ifdef CADEBUG2 + #undef CADEBUG2 + #endif + #ifdef CADEBUG3 + #undef CADEBUG3 + #endif + #if GPUCA_CADEBUG >= 1 && !defined(GPUCA_GPUCODE) + #if GPUCA_CADEBUG >= 2 + #define CADEBUG(...) if (CADEBUG_CONDITION) {__VA_ARGS__;} + #define CADEBUG2(cmd, ...) if (CADEBUG_CONDITION) {__VA_ARGS__; cmd;} + #define GPUCA_CADEBUG_ENABLED 2 + #else + #define CADEBUG(...) __VA_ARGS__ + #define CADEBUG2(cmd, ...) {__VA_ARGS__; cmd;} + #define GPUCA_CADEBUG_ENABLED 1 + #endif + #define CADEBUG3(...) __VA_ARGS__ #endif #undef GPUCA_CADEBUG #endif @@ -81,5 +94,6 @@ #ifndef CADEBUG #define CADEBUG(...) #define CADEBUG2(cmd, ...) {cmd;} + #define CADEBUG3(...) #endif // clang-format on diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 26ffff549bad2..f0183cd5472b6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -17,6 +17,7 @@ // #define GPUCA_CADEBUG_ENABLED #include "GPUCommonDef.h" +#include "GPUDef.h" #if !defined(GPUCA_GPUCODE) && (defined(GPUCA_MERGER_BY_MC_LABEL) || defined(GPUCA_CADEBUG_ENABLED) || GPUCA_MERGE_LOOPER_MC) #include "AliHLTTPCClusterMCData.h" diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a8a71f124ab49..83cce0f9c4bfe 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -13,7 +13,6 @@ /// \author David Rohr, Sergey Gorbunov #define GPUCA_CADEBUG 0 -#define DEBUG_SINGLE_TRACK -1 // #define DEBUG_REBUILD_MC #include "GPUTPCDef.h" @@ -41,6 +40,7 @@ #include #include #include "AliHLTTPCClusterMCData.h" +static thread_local bool CADEBUG_CONDITION = true; #endif #ifndef GPUCA_GPUCODE @@ -873,7 +873,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t const float kFactor = merger.GetConstantMem()->tpcTrackers[0].GetChiSeedFactor() * 4.f; maxDistY2 = (maxDistY2 + mC[0]) * kFactor; maxDistZ2 = (maxDistZ2 + mC[2]) * kFactor; - auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG(, this)](int32_t ih, float y, float z) { + auto chkFunction = [clusters, markReject, maxDistY2, maxDistZ2, projY, projZ, noReject CADEBUG3(, this)](int32_t ih, float y, float z) { float dy = y - projY; float dz = z - projZ; if (!noReject && (dy * dy > maxDistY2 || dz * dz > maxDistZ2)) { @@ -1338,7 +1338,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() return; } - CADEBUG(if (DEBUG_SINGLE_TRACK != -1) { static const std::set dbgTracks = []() { std::set m; if (const char* e = DEBUG_SINGLE_TRACK == -2 ? getenv("DEBUG_TRACK") : nullptr) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } else { m.insert(DEBUG_SINGLE_TRACK); } return m; }(); if (!dbgTracks.count(iTrk)) { track.SetNClusters(0); track.SetOK(0); return; } }); + CADEBUG3(if (GPUCA_CADEBUG_ENABLED >= 2) { static const std::set dbgTracks = []() { std::set m; if (const char* e = getenv("DEBUG_TRACK")) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } return m; }(); CADEBUG_CONDITION = dbgTracks.count(iTrk); }); int32_t nTrackHits = track.NClusters(); int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut diff --git a/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py index 01ff969cf17a6..03d863dbb55ef 100755 --- a/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py +++ b/GPU/GPUTracking/Standalone/scripts/compareTrackNcl.py @@ -11,7 +11,7 @@ # or submit itself to any jurisdiction. # Usage: compareTrackNcl.py FILE1 FILE2, with files of lines "Track NCl " (from --DEBUGmergerDumpTrackNCl). -# Prints the ids of tracks with less than half the clusters in FILE1 compared to FILE2 as DEBUG_TRACK="...", to be used as environment variable with DEBUG_SINGLE_TRACK -2 in GPUTPCGMTrackParam.cxx, followed by some statistics. +# Prints the ids of tracks with less than half the clusters in FILE1 compared to FILE2 as DEBUG_TRACK="...", to be used as environment variable with GPUCA_CADEBUG 2 in GPUTPCGMTrackParam.cxx, followed by some statistics. import sys From b6c1037d24a12cb14eb5a75177e6ef0ac8cba40f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 17:11:57 +0100 Subject: [PATCH 33/54] GPU TPC: Make extrapolation more robust by stopping early when there is no chance of continuing --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index fbedbd854833b..f9af92eb90f9b 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -106,6 +106,8 @@ AddOptionRTC(trackFitMinHitsMaxQPtIncrease, float, 0.2f, "", 0, "For the QPt-dep AddOptionRTC(rebuildTrackMaxSharedFraction, float, 0.1f, "", 0, "Max fraction of shared clusters for rebuilt tracks") AddOptionRTC(rebuildTrackMaxNonIntCov, float, 4.f, "", 0, "Max Err2 allowed for non-interpolated cluster attachment during rebuild") AddOptionRTC(rebuildErrorWeightFactor, float, 5.f, "", 0, "Factor for the error (distance^2) of a cluster candidate in its hit weight 1 / (1 + factor * error) during rebuild") +AddOptionRTC(maxSinPhiForFitUpdate, float, 0.9396926f, "", 0, "Max abs(sinPhi) of the track to update it with a cluster in the track fit)") // default sin(70 deg = 0.9396926 +AddOptionRTC(rebuildTrackMaxQPtDrift, float, 0.5f, "", 0, "Stop the track extrapolation during rebuild when QPt changed by more than this fraction of max(1, abs(QPt)) compared to before the extrapolation (0 = disabled)") AddOptionRTC(rebuildTrackExtrMaxAvgChi2, float, 4.f, "", 0, "Max average chi2 per cluster to accept an extrapolated segment (0 = disabled)") AddOptionRTC(rebuildTrackFallbackErr, float, 1.f, "", 0, "Error (distance^2) assigned to the original clusters of a track used as fallback candidates in rows that could not be searched during rebuild") AddOptionRTC(rebuildTrackFallbackTrackChi2, float, 4.f, "", 0, "Chi2 per cluster assumed for the hit weight of tracks without successful fit (NDF <= 0) during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 83cce0f9c4bfe..808f18fe53858 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -59,9 +59,9 @@ using namespace o2::tpc; GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; - const float maxSinForUpdate = CAMath::Sin(70.f * CAMath::Deg2Rad()); const GPUParam& GPUrestrict() param = merger.Param(); + const float maxSinForUpdate = param.rec.tpc.maxSinPhiForFitUpdate; GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); GPUdEdx dEdx, dEdxAlt; @@ -302,6 +302,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow, lastFoundRow = lastPropagateRow; uint8_t consecGoodRows = ihitStart == ((iWay & 1) ? 0 : maxN - 1) ? param.rec.tpc.rebuildTrackExtrMinConsecGoodRows : 0, consecGoodRowsMissing = 0, segmentRows = 0; // Continue the track directly only if its last hit was accepted in the fit float segmentStartChi2 = mChi2; + const float qPtStart = mP[4]; prop.SetTrack(this, prop.GetAlpha()); uint32_t iExtrapolatedRow = lastPropagateRow + up; for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers @@ -317,7 +318,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ fail = true; break; } - if (CAMath::Abs(mP[2]) > maxSinForUpdate) { + if (CAMath::Abs(mP[2]) > maxSinForUpdate || CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate || mC[0] >= param.rec.tpc.rebuildTrackMaxNonIntCov || mC[2] >= param.rec.tpc.rebuildTrackMaxNonIntCov) { // No further clusters can be attached to the fit fail = true; break; } @@ -380,6 +381,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); + if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else + candidate.best = -1; + break; + } if (retValUpd != 0) { candidate.best = -1; if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { @@ -426,9 +431,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ candidate.best = -1; } } - } - - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped + StoreLoopPropagation(merger, sector, iExtrapolatedRow, iTrk, up == 1, prop.GetAlpha()); + CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); + } + } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } From 006f5d4f3da8191f2d5ec65809e47a9875d6a212 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 17:47:22 +0100 Subject: [PATCH 34/54] GPU TPC: revert state after extrapolation in case of problems --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index f9af92eb90f9b..e43c754eba044 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -183,6 +183,7 @@ AddOptionRTC(tubeExtraProtectEdgePads, uint8_t, 2, "", 0, "Increase Protection, AddOptionRTC(rebuildTrackExtrMaxMissingRows, uint8_t, 20, "", 0, "Maximum total number of rows allowed to be missing during track extrapolation") AddOptionRTC(rebuildTrackExtrMinConsecGoodRows, uint8_t, 8, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") AddOptionRTC(rebuildTrackExtrMinConsecGoodRowsEdge, uint8_t, 3, "", 0, "Minimum number of consecutive rows required to accept extrapolated segment") +AddOptionRTC(rebuildTrackExtrMaxRowsBack, uint8_t, 3, "", 0, "After the track extrapolation, propagate the track back to the last accepted cluster if it is at most this number of rows away, otherwise revert to the track before the extrapolation") AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionRTC(rebuildTrackCandidateErrChi2, uint8_t, 0, "", 0, "Rank cluster candidates during rebuild by chi2 instead of distance^2 (rebuildTrackAltCandidateGoodErr is then in units of chi2)") AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 808f18fe53858..a2d16eead1db6 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -302,7 +302,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t lastGoodRow = lastPropagateRow, lastExtrapolateRow = lastPropagateRow, lastFoundRow = lastPropagateRow; uint8_t consecGoodRows = ihitStart == ((iWay & 1) ? 0 : maxN - 1) ? param.rec.tpc.rebuildTrackExtrMinConsecGoodRows : 0, consecGoodRowsMissing = 0, segmentRows = 0; // Continue the track directly only if its last hit was accepted in the fit float segmentStartChi2 = mChi2; - const float qPtStart = mP[4]; + const float qPtStart = mP[4], alphaStart = prop.GetAlpha(); + const GPUTPCGMTrackParam paramStart = *this; // To revert to if the extrapolation fails + float lastFoundX = 0.f, lastFoundAlpha = 0.f; + bool revert = false; prop.SetTrack(this, prop.GetAlpha()); uint32_t iExtrapolatedRow = lastPropagateRow + up; for (; iExtrapolatedRow < GPUTPCGeometry::NROWS; iExtrapolatedRow += up) { // uint8_t implies > 0! // TODO: Try to reduce some variables to int8/uint8 to save registers @@ -310,12 +313,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (int32_t iAttempt = 0; iAttempt < 2; iAttempt++) { float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iExtrapolatedRow), tmpY, tmpZ)) { - fail = true; + fail = revert = true; break; } merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoX(sector, iExtrapolatedRow, tmpY, tmpZ, tmpX); if (prop.PropagateToXAlpha(tmpX, param.Alpha(sector), inFlyDirection)) { - fail = true; + fail = revert = true; break; } if (CAMath::Abs(mP[2]) > maxSinForUpdate || CAMath::Abs(prop.GetSinPhi0()) >= maxSinForUpdate || mC[0] >= param.rec.tpc.rebuildTrackMaxNonIntCov || mC[2] >= param.rec.tpc.rebuildTrackMaxNonIntCov) { // No further clusters can be attached to the fit @@ -338,7 +341,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } if (prop.RotateToAlpha(param.Alpha(sector))) { // GetPropagatedYZ / InverseTransformYZtoX of the next attempt need Y in the frame of the new sector - fail = true; + fail = revert = true; break; } } @@ -365,6 +368,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { candidate.best = -1; + revert = true; break; } const float time = merger.GetConstantMem()->ioPtrs.clustersNative ? cl.getTime() : -1.f; // TODO: When is it possible that we do not have clusterNative? @@ -383,11 +387,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else candidate.best = -1; + revert = true; // The rejected cluster is already in the fit break; } if (retValUpd != 0) { candidate.best = -1; if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { + revert = true; break; } found = false; // Cluster rejected by chi2, handle as missing row @@ -397,6 +403,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } if (found) { lastFoundRow = iExtrapolatedRow; + lastFoundX = mX; + lastFoundAlpha = prop.GetAlpha(); if (++consecGoodRows >= param.rec.tpc.rebuildTrackExtrMinConsecGoodRows) { if (param.rec.tpc.rebuildTrackExtrMaxAvgChi2 <= 0 || mChi2 - segmentStartChi2 <= param.rec.tpc.rebuildTrackExtrMaxAvgChi2 * segmentRows) { lastGoodRow = iExtrapolatedRow; // TODO: Could optionally store the track state here, and revert to the good state in case we reject further clusters @@ -435,6 +443,11 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ StoreLoopPropagation(merger, sector, iExtrapolatedRow, iTrk, up == 1, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); } + const int32_t rowsMoved = CAMath::Abs((int32_t)(iExtrapolatedRow < GPUTPCGeometry::NROWS ? iExtrapolatedRow : iExtrapolatedRow - up) - (int32_t)lastGoodRow); + if (revert || lastGoodRow == lastPropagateRow || lastFoundRow != lastGoodRow || (iWay == nWays - 3 && sector != lastSector) || rowsMoved > param.rec.tpc.rebuildTrackExtrMaxRowsBack || (rowsMoved && prop.PropagateToXAlpha(lastFoundX, lastFoundAlpha, !inFlyDirection))) { // Failure, nothing accepted, rejected clusters in the fit, sector changed before the next way goes back, or too far from the last accepted cluster to go back to it + *this = paramStart; + prop.SetTrack(this, alphaStart); + } } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); From 1136df49af511c519a5f3060a1b943f4e0c212c6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 21:50:00 +0100 Subject: [PATCH 35/54] GPU TPC: Fix some cases of fit failues after track rebuild (bad initial state, better handling of high inclination flags, fallback clusters for short tracks) --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 + GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 37 +++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index e43c754eba044..a5e1fbd7f651c 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -186,6 +186,8 @@ AddOptionRTC(rebuildTrackExtrMinConsecGoodRowsEdge, uint8_t, 3, "", 0, "Minimum AddOptionRTC(rebuildTrackExtrMaxRowsBack, uint8_t, 3, "", 0, "After the track extrapolation, propagate the track back to the last accepted cluster if it is at most this number of rows away, otherwise revert to the track before the extrapolation") AddOptionRTC(rebuildTrackExtrConsecGoodRowsMaxGap, uint8_t, 1, "", 0, "Max row gap size to consider the segment as consecutive rows") AddOptionRTC(rebuildTrackCandidateErrChi2, uint8_t, 0, "", 0, "Rank cluster candidates during rebuild by chi2 instead of distance^2 (rebuildTrackAltCandidateGoodErr is then in units of chi2)") +AddOptionRTC(rebuildTrackMaxRowsFromEnd, uint8_t, 10, "", 0, "If the last fitted hit of the outward way of the rebuild is more than this number of rows away from the end of the track, the final fit starts from the original track parameters") +AddOptionRTC(rebuildTrackFallbackMaxNCl, uint8_t, 20, "", 0, "For tracks with at most this number of clusters, keep the original cluster of a row as candidate during rebuild if no candidate was found but it passes the interpolated chi2 cut (0 = disabled)") AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC AddHelp("help", 'h') diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a2d16eead1db6..2b9831f3a2879 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -244,8 +244,19 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } else { + if (!markHighIncl && nWays >= 2 && CAMath::Abs(mP[2]) < maxSinPhi && CAMath::Max(CAMath::Abs(yy - mP[0]), CAMath::Abs(zz - mP[1])) <= 5.f) { // abs(sinPhi) decreases in the direction we are going (we come from another sector) and the cluster is close: continue from the cluster position, a normal update could distort sinPhi and QPt at this inclination + float err2Y, err2Z; + prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, -1.f, 0.f, 0.f); + prop.Model().Y() = mP[0] = yy; // Like an update that takes the position fully from the cluster and leaves the other parameters untouched + prop.Model().Z() = mP[1] = zz; + mC[0] = err2Y; + mC[2] = err2Z; + mC[1] = mC[3] = mC[4] = mC[6] = mC[7] = mC[10] = mC[11] = 0.f; + lastUpdateX = mX; + } else { + nMissed2++; + } MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; CADEBUG(printf(" --- break-sinphi\n")); NTolerated++; if (param.rec.tpc.rebuildTrackInFit && markHighIncl) { @@ -267,8 +278,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); - if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { - CADEBUG(printf("Reinit linearization\n")); + if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { // TODO: Could be investigated to reinit the linearization also when sinPhi drifts away from it (a reinit when QPt differs makes things worse, early in the way the QPt of the linearization is usually the better one) + CADEBUG(printf("Reinit linearization: way %d, QPt %7.2f (%7.2f), SP %5.2f (%5.2f), NDF %d\n", iWay, mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mNDF)); prop.SetTrack(this, prop.GetAlpha()); resetT0 = 20; } @@ -277,6 +288,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ interpolatedStart = interpolationIndex; N++; covYYUpd = mC[0]; + if (param.rec.tpc.rebuildTrackInFit && !rebuilt) { // The high inclination row range is valid only if it lasts until the end of the way, reset it since this later hit was fitted + auto& highInclRow = ((clusters[0].row > clusters[maxN - 1].row) ^ (iWay & 1)) ? merger.TrackRebuildHelper()[iTrk].highInclRowLow : merger.TrackRebuildHelper()[iTrk].highInclRowHigh; + if (highInclRow != 255) { + highInclRow = 255; + } + } } else if (retValHit == 1) { if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); @@ -457,6 +474,14 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ Alpha = prop.GetAlpha(); if (ihitStart != 0) { MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); + if (CAMath::Abs(clusters[ihitStart].row - clusters[0].row) > param.rec.tpc.rebuildTrackMaxRowsFromEnd) { // The fit ended far away from the end of the track, the final fit cannot start from there, so let it start from the original parameters like the first way + const int32_t ndf = mNDF; + const float chi2 = mChi2; + *this = track.GetParam(); + mNDF = ndf; + mChi2 = chi2; + Alpha = track.GetAlpha(); + } } return true; } @@ -568,6 +593,12 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } else { rejectChi2 = true; } + } else if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { + const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation + candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; + } } } else if (iWay == nWays - 1) { if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { From c6c70318412b81a63e3ca612a92d32acc71eff98 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 12 Nov 2026 22:45:47 +0100 Subject: [PATCH 36/54] GPU TPC: Fix deltaTimeFwd/deltaTimeBwd after track rebuild --- GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx | 8 ++-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 45 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 1 + 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx index ddd6862b96bbb..b2009aba09d41 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMO2Output.cxx @@ -215,9 +215,11 @@ GPUdii() void GPUTPCGMO2Output::Thread(int32_t nBlocks if (track.PrevSegment() >= 0) { const GPUTPCGMMergedTrack* chkTrk = track.GetFirstSegment(tracks, merger.Param().rec.enableCyclicGraphWorkarounds); - const auto& firstPrevCluster = trackClusters[chkTrk->FirstClusterRef()]; - t1 = clusters->clustersLinear[firstPrevCluster.num].getTime(); - sector1 = firstPrevCluster.sector; + if (chkTrk->NClusters()) { // First segment can have lost all its clusters in the rebuild, then its cluster reference is invalid + const auto& firstPrevCluster = trackClusters[chkTrk->FirstClusterRef()]; + t1 = clusters->clustersLinear[firstPrevCluster.num].getTime(); + sector1 = firstPrevCluster.sector; + } } bool cce = track.CCE() && ((sector1 < MAXSECTOR / 2) ^ (sector2 < MAXSECTOR / 2)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2b9831f3a2879..234f20d8a2c7d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -81,6 +81,10 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t nWays = param.rec.tpc.nWays; const int32_t maxN = N; + if (rebuilt && maxN > 0 && param.par.continuousTracking && !track.CCE() && !track.Looper()) { // The rebuilt track can have other first / last clusters than the track the Z shift was computed for + const float t0 = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[0].num].getTime(), tN = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[clusters[maxN - 1].num].getTime(); + ConstrainZ(merger, clusters[0].sector, CAMath::Max(t0, tN), CAMath::Min(t0, tN)); + } int32_t ihitStart = 0; int32_t interpolatedStart = 0; float covYYUpd = 0.f; @@ -1327,28 +1331,29 @@ GPUd() float GPUTPCGMTrackParam::ShiftZ(const GPUTPCGMMerger& GPUrestrict() merg merger.GetConstantMem()->calibObjects.fastTransform->TransformIdealZ(sector, cltmax, basez, mTOffset); deltaZ = basez - refZ; } - { - float deltaT = merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + mTOffset += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaZtoDeltaTimeInTimeFrame(sector, deltaZ); + return -ConstrainZ(merger, sector, cltmax, cltmin, deltaZ); +} + +GPUd() float GPUTPCGMTrackParam::ConstrainZ(const GPUTPCGMMerger& GPUrestrict() merger, uint32_t sector, float cltmax, float cltmin, float deltaZ) +{ + const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); + const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); + // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); + float deltaT = 0.f; + if (mTOffset < minT) { + deltaT = minT - mTOffset; + } + if (mTOffset + deltaT > maxT) { + deltaT = maxT - mTOffset; + } + if (deltaT != 0.f) { + deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); + // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); mTOffset += deltaT; - const float maxT = cltmin - merger.GetConstantMem()->calibObjects.fastTransform->getT0(); - const float minT = cltmax - merger.GetConstantMem()->calibObjects.fastTransform->getMaxDriftTime(sector); - // printf("T Check: Clusters %f %f, min %f max %f vtx %f\n", tz1, tz2, minT, maxT, mTOffset); - deltaT = 0.f; - if (mTOffset < minT) { - deltaT = minT - mTOffset; - } - if (mTOffset + deltaT > maxT) { - deltaT = maxT - mTOffset; - } - if (deltaT != 0.f) { - deltaZ += merger.GetConstantMem()->calibObjects.fastTransform->convDeltaTimeToDeltaZinTimeFrame(sector, deltaT); - // printf("Moving clusters to TPC Range: QPt %f, New mTOffset %f, t1 %f, t2 %f, Shift %f in Z: %f to %f --> %f to %f in T\n", mP[4], mTOffset + deltaT, tz1, tz2, deltaZ, tz2 - mTOffset, tz1 - mTOffset, tz2 - mTOffset - deltaT, tz1 - mTOffset - deltaT); - mTOffset += deltaT; - } - mP[1] -= deltaZ; } - // printf("\n"); - return -deltaZ; + mP[1] -= deltaZ; + return deltaZ; } GPUd() bool GPUTPCGMTrackParam::CheckCov() const diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 4619db960981d..1d98920cfb094 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -203,6 +203,7 @@ class GPUTPCGMTrackParam GPUd() void Rotate(float alpha); GPUd() float ShiftZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float clx); GPUd() float ShiftZ(const GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, int32_t N); + GPUd() float ConstrainZ(const GPUTPCGMMerger& merger, uint32_t sector, float cltmax, float cltmin, float deltaZ = 0.f); // Shift Z / TOffset such that the clusters are within the TPC drift volume, applies in addition the pending shift deltaZ to Z, returns the total shift GPUd() static float Reciprocal(float x) { return 1.f / x; } GPUdi() static void Assign(float& x, bool mask, float v) From eed55d68cfcdbb37ef74671d31a22e30f67f4890 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 13 Nov 2026 17:24:49 +0100 Subject: [PATCH 37/54] GPU TPC: Fix storing OuterParam after implementing track rebuild --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 234f20d8a2c7d..2852a52b637ac 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -96,15 +96,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; - if (iWay && (iWay & 1) == 0) { - StoreOuter(&track.OuterParam(), prop.GetAlpha()); // TODO: Fix outer param! - } - int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); const bool refit = (nWays == 1 || iWay >= 1); const bool finalOutInFit = iWay + 2 >= nWays; const bool finalFit = iWay == nWays - 1; + if (iWay && finalFit) { + StoreOuter(&track.OuterParam(), rebuilt ? Alpha : prop.GetAlpha()); + } + ResetCovariance(); prop.SetSeedingErrors(!(refit && !retryAttempt)); prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed From e3239be3ce9e117a9663ed60bd33674d2b6260ca Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 13 Nov 2026 23:42:06 +0100 Subject: [PATCH 38/54] GPU TPC: Option to run interpolation-rejection after track rebuild --- .../reconstruction/test/testGPUCATracking.cxx | 1 - GPU/GPUTracking/Base/GPUParam.cxx | 1 + GPU/GPUTracking/Definitions/GPUSettingsList.h | 11 ++- GPU/GPUTracking/Global/GPUChainTracking.cxx | 12 --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 92 ++++++++++--------- 5 files changed, 55 insertions(+), 62 deletions(-) diff --git a/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx b/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx index 9d44e5c8b5890..f93f43e1cd258 100644 --- a/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx +++ b/Detectors/TPC/reconstruction/test/testGPUCATracking.cxx @@ -64,7 +64,6 @@ BOOST_AUTO_TEST_CASE(CATracking_test1) config.configGRP.solenoidBzNominalGPU = solenoidBz; config.configGRP.grpContinuousMaxTimeBin = continuous ? GPUSettings::TPC_MAX_TF_TIME_BIN : 0; // Number of timebins in timeframe if continuous, 0 otherwise - config.configReconstruction.tpc.nWays = 3; // Should always be 3! config.configReconstruction.tpc.searchWindowDZDR = 2.5f; // Should always be 2.5 for looper-finding and/or continuous tracking config.configReconstruction.tpc.trackReferenceX = refX; diff --git a/GPU/GPUTracking/Base/GPUParam.cxx b/GPU/GPUTracking/Base/GPUParam.cxx index 6650248439626..b7f8ccb9fa668 100644 --- a/GPU/GPUTracking/Base/GPUParam.cxx +++ b/GPU/GPUTracking/Base/GPUParam.cxx @@ -113,6 +113,7 @@ void GPUParam::UpdateSettings(const GPUSettingsGRP* g, const GPUSettingsProcessi if (d) { rec.dyn = *d; } + rec.tpc.nWays = 1 + 2 * ((rec.tpc.mergerInitial3wayFit > 0) + (rec.tpc.mergerInterpolateErrors > 0) + (rec.tpc.rebuildTrackInFit > 0)); } void GPUParam::UpdateBzOnly(float newSolenoidBz, bool assumeConstantBz) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index a5e1fbd7f651c..c31c12f8c945f 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -148,8 +148,11 @@ AddOptionRTC(cfMinSplitNum, uint8_t, 1, "", 0, "Minimum number of split charges AddOptionRTC(cfNoiseSuppressionEpsilon, uint8_t, 10, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression") AddOptionRTC(cfNoiseSuppressionEpsilonRelative, uint8_t, 76, "", 0, "Cluster Finder: Difference between peak and charge for the charge to count as a minima during noise suppression, relative as fraction of 255") AddOptionRTC(cfEdgeTwoPads, uint8_t, 0, "", 0, "Flag clusters with peak on the 2 pads closes to the sector edge as edge cluster") -AddOptionRTC(nWays, uint8_t, 3, "", 0, "Do N fit passes in final fit of merger (must be odd to end with inward fit)") -AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions") +AddOptionRTC(mergerInitial3wayFit, uint8_t, 0, "", 0, "The track fit does 2 initial fit passes for a good track state before running the rebuild or the interpolation rejection, ads 2 fit iteration") +AddOptionRTC(mergerInterpolateErrors, uint8_t, 0, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection, adds 2 fit iterations") +AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") +AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") +AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions, adds 2 fit iterations") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history @@ -166,9 +169,6 @@ AddOptionRTC(compressionSortOrder, uint8_t, o2::gpu::GPUSettings::SortTime, "", AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for TPC cluster charge in compression mode 1") AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") -AddOptionRTC(mergerInterpolateErrors, uint8_t, 1, "", 0, "Use interpolation instead of extrapolation for chi2 based cluster rejection") -AddOptionRTC(mergerInterpolateRejectAlsoOnCurrentPosition, uint8_t, 1, "", 0, "When using mergerInterpolateErrors, reject based on chi2 twice computed with interpolated and current track position starting from NDF > mergerNonInterpolateRejectMinNDF") -AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF of track for non-interpolated reject (both for chi2 and absolute distance)") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") @@ -190,6 +190,7 @@ AddOptionRTC(rebuildTrackMaxRowsFromEnd, uint8_t, 10, "", 0, "If the last fitted AddOptionRTC(rebuildTrackFallbackMaxNCl, uint8_t, 20, "", 0, "For tracks with at most this number of clusters, keep the original cluster of a row as candidate during rebuild if no candidate was found but it passes the interpolated chi2 cut (0 = disabled)") AddOptionRTC(rebuildTrackNonIntMinNDF, uint8_t, 10, "", 0, "NDF the track must exceed for non-interpolated cluster attachment (one-sided search and extrapolation) during rebuild") AddOptionArray(PID_remap, int8_t, 9, (0, 1, 2, 3, 4, 5, 6, 7, 8), "", 0, "Remap Ipid to PID_reamp[Ipid] (no remap if<0)") // BUG: CUDA cannot yet handle AddOptionArrayRTC +AddVariableRTC(nWays, uint8_t, 0) AddHelp("help", 'h') EndConfig() diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index df74187a4f9d5..2761fcb3ec733 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -258,18 +258,6 @@ bool GPUChainTracking::ValidateSteps() bool GPUChainTracking::ValidateSettings() { int32_t gatherMode = mRec->GetProcessingSettings().tpcCompressionGatherMode == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCCompression).par_COMP_GATHER_MODE : mRec->GetProcessingSettings().tpcCompressionGatherMode; - if ((param().rec.tpc.nWays & 1) == 0) { - GPUError("nWay setting musst be odd number!"); - return false; - } - if (param().rec.tpc.mergerInterpolateErrors && param().rec.tpc.nWays < 3) { - GPUError("Cannot do error interpolation with NWays < 3!"); - return false; - } - if (param().rec.tpc.rebuildTrackInFit && !param().rec.tpc.mergerInterpolateErrors) { - GPUError("Need error interpolation to rebuild tracks during fit"); - return false; - } if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) { GPUError("Need at least 1 cluster candidate to rebuild tracks during fit"); return false; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2852a52b637ac..97296e6808cbf 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -70,13 +70,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetMaterialTPC(); prop.SetPolynomialField(¶m.polynomialField); prop.SetMaxSinPhi(maxSinPhi); - if (param.rec.tpc.mergerInterpolateErrors && !rebuilt) { - for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range + if ((param.rec.tpc.rebuildTrackInFit && !rebuilt) || param.rec.tpc.mergerInterpolateErrors) { // Both the rebuild ways and the interpolation-rejection ways fill the interpolation + for (uint32_t i = 0; i < interpolation.size; i++) { // TODO: Tune the zeroing range interpolation.hit[i].markInvalid(); } - if (param.rec.tpc.rebuildTrackInFit) { - merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; - } + } + if (param.rec.tpc.rebuildTrackInFit && !rebuilt) { + merger.TrackRebuildHelper()[iTrk].highInclRowLow = merger.TrackRebuildHelper()[iTrk].highInclRowHigh = 255; } const int32_t nWays = param.rec.tpc.nWays; @@ -89,20 +89,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; + const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; - // TODO: Could do 5 ways and interpolate again on the rebuilt clusters and reject in the final way based on that (and get the outer param from the rebuilt clusters), but costs 2 more ways - for (int32_t iWay = rebuilt ? nWays - 1 : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? + for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissed2 = 0; float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); const bool refit = (nWays == 1 || iWay >= 1); - const bool finalOutInFit = iWay + 2 >= nWays; + const bool finalOutInFit = iWay >= nWays - 2; const bool finalFit = iWay == nWays - 1; if (iWay && finalFit) { - StoreOuter(&track.OuterParam(), rebuilt ? Alpha : prop.GetAlpha()); + StoreOuter(&track.OuterParam(), rebuilt && iWay == wayRebuildStart + 2 ? Alpha : prop.GetAlpha()); // In the first way of the rebuilt pass, prop is not yet initialized } ResetCovariance(); @@ -110,7 +110,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ prop.SetFitInProjections(true); // param.rec.fitInProjections == -1 ? (iWay == 0) : param.rec.fitInProjections); // TODO: Reenable once fixed prop.SetPropagateBzOnly(param.rec.fitPropagateBzOnly == -1 ? !finalFit : param.rec.fitPropagateBzOnly); prop.SetMatLUT((param.rec.useMatLUT && finalFit) ? merger.GetConstantMem()->calibObjects.matLUT : nullptr); - prop.SetTrack(this, iWay && !rebuilt ? prop.GetAlpha() : Alpha); + prop.SetTrack(this, iWay && (!rebuilt || iWay > wayRebuildStart + 2) ? prop.GetAlpha() : Alpha); ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); @@ -141,7 +141,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t clusterState = clusters[ihit].state; const float clAlpha = param.Alpha(clusters[ihit].sector); float xx, yy, zz; - const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, param.rec.tpc.rebuildTrackInFit ? rebuilt : allowChangeClusters); + const int32_t currentClusterStatus = MergeDoubleRowClusters(ihit, wayDirection, clusters, merger, prop, xx, yy, zz, maxN, clAlpha, clusterState, (param.rec.tpc.rebuildTrackInFit && !param.rec.tpc.mergerInterpolateErrors) ? rebuilt : allowChangeClusters); // TODO: Check about tracks who have clusters in the same row multiple times in different sectors const auto& cluster = clusters[ihit]; @@ -149,7 +149,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); - if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic + if (rebuilt) { + interpolationIndex = ihit; + } else if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex + rowGap * wayDirection, interpolation.size); fitBroken = true; break; @@ -161,7 +163,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); const bool crossCE = lastSector != 255 && ((cluster.sector >= GPUTPCGeometry::NSECTORS / 2) != (lastSector >= GPUTPCGeometry::NSECTORS / 2)); const bool ceOverlap = crossCE && ((cluster.row > lastPropagateRow) != ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1))); // Row gap goes backwards at the CE crossing, these rows were already searched on both sides - const uint8_t doInterpolate = ((param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 3) ? 1 : ((param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) ? 2 : 0)); + const uint8_t doInterpolate = (!param.rec.tpc.rebuildTrackInFit || (param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : (iWay == wayRebuildStart ? 1 : (iWay == wayRebuildStart + 1 ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; @@ -181,7 +183,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } - interpolationIndex += rowGap * wayDirection; + if (!rebuilt) { + interpolationIndex += rowGap * wayDirection; + } } lastPropagateRow = cluster.row; @@ -195,7 +199,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissed2++; NTolerated++; - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); } continue; @@ -205,7 +209,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t%21sPropaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Res %8.3f %8.3f --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - PErr %d\n", "", prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[0] - yy, mP[1] - zz, sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValProp)); // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } fitBroken = true; @@ -214,7 +218,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == nWays - 2 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + if (iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps } if (allowChangeClusters) { @@ -231,7 +235,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ HandleCrossCE(param, cluster.sector, lastSector); lastSector = cluster.sector; - if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) { + if ((param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 3) || (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart)) { prop.InterpolateFill(&inter); } @@ -248,7 +252,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Reinit linearization\n")); prop.SetTrack(this, prop.GetAlpha()); } else { - if (!markHighIncl && nWays >= 2 && CAMath::Abs(mP[2]) < maxSinPhi && CAMath::Max(CAMath::Abs(yy - mP[0]), CAMath::Abs(zz - mP[1])) <= 5.f) { // abs(sinPhi) decreases in the direction we are going (we come from another sector) and the cluster is close: continue from the cluster position, a normal update could distort sinPhi and QPt at this inclination + if (!markHighIncl && nWays > 1 && CAMath::Abs(mP[2]) < maxSinPhi && CAMath::Max(CAMath::Abs(yy - mP[0]), CAMath::Abs(zz - mP[1])) <= 5.f) { // abs(sinPhi) decreases in the direction we are going (we come from another sector) and the cluster is close: continue from the cluster position, a normal update could distort sinPhi and QPt at this inclination float err2Y, err2Z; prop.GetErr2(err2Y, err2Z, param, zz, cluster.row, clusterState, cluster.sector, -1.f, 0.f, 0.f); prop.Model().Y() = mP[0] = yy; // Like an update that takes the position fully from the cluster and leaves the other parameters untouched @@ -299,7 +303,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } else if (retValHit == 1) { - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); } fitBroken = true; @@ -307,7 +311,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } lastUpdateRow = cluster.row; - assert(!param.rec.tpc.mergerInterpolateErrors || rebuilt || iWay != nWays - 2 || ihit || interpolationIndex == 0); + assert(((!param.rec.tpc.rebuildTrackInFit || iWay != wayRebuildStart + 1) && (!param.rec.tpc.mergerInterpolateErrors || iWay != nWays - 2)) || ihit || interpolationIndex == 0); } if (!(iWay & 1) && !finalFit && !track.CCE() && !track.Looper()) { deltaZ = ShiftZ(clusters, merger, maxN); @@ -315,7 +319,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ deltaZ = 0.f; } - if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= nWays - 3 && !(fitBroken && iWay == nWays - 2) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255 && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF) { + if (param.rec.tpc.rebuildTrackInFit && !rebuilt && !(param.rec.tpc.disableRebuildAttachment & 16) && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && iWay >= wayRebuildStart && !(fitBroken && iWay == wayRebuildStart + 1) && CAMath::Abs(mP[2]) < maxSinForUpdate && lastUpdateRow != 255 && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF) { const int32_t up = ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1)) ? 1 : -1; int32_t sector = lastSector; CADEBUG(if (merger.MergedTrackMC()) printf("Extrapolate Start Track %d - sector %2d row %3d %s - fake %d\n", iTrk, sector, (int32_t)lastPropagateRow, up == 1 ? "upwards" : "downwards", (int)merger.MergedTrackMC()[iTrk].isFake())); @@ -465,7 +469,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); } const int32_t rowsMoved = CAMath::Abs((int32_t)(iExtrapolatedRow < GPUTPCGeometry::NROWS ? iExtrapolatedRow : iExtrapolatedRow - up) - (int32_t)lastGoodRow); - if (revert || lastGoodRow == lastPropagateRow || lastFoundRow != lastGoodRow || (iWay == nWays - 3 && sector != lastSector) || rowsMoved > param.rec.tpc.rebuildTrackExtrMaxRowsBack || (rowsMoved && prop.PropagateToXAlpha(lastFoundX, lastFoundAlpha, !inFlyDirection))) { // Failure, nothing accepted, rejected clusters in the fit, sector changed before the next way goes back, or too far from the last accepted cluster to go back to it + if (revert || lastGoodRow == lastPropagateRow || lastFoundRow != lastGoodRow || (iWay == wayRebuildStart && sector != lastSector) || rowsMoved > param.rec.tpc.rebuildTrackExtrMaxRowsBack || (rowsMoved && prop.PropagateToXAlpha(lastFoundX, lastFoundAlpha, !inFlyDirection))) { // Failure, nothing accepted, rejected clusters in the fit, sector changed before the next way goes back, or too far from the last accepted cluster to go back to it *this = paramStart; prop.SetTrack(this, alphaStart); } @@ -474,7 +478,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } - if (param.rec.tpc.rebuildTrackInFit && iWay == nWays - 2) { + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { Alpha = prop.GetAlpha(); if (ihitStart != 0) { MarkClusters(clusters, 0, ihitStart - 1, 1, GPUTPCGMMergedTrackHit::flagHighIncl); @@ -567,14 +571,16 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; + const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; const int32_t wayDirection = (iWay & 1) ? -1 : 1; + const bool finalOutInFit = iWay >= nWays - 2; const auto& cluster = clusters[ihit]; int32_t retValUpd = 0, retValInt = 0; float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; - if (param.rec.tpc.rebuildTrackInFit) { + if (param.rec.tpc.rebuildTrackInFit && (finalOutInFit || iWay < wayRebuildStart + 2)) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds } } else { @@ -589,27 +595,25 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger bool rejectChi2 = !retryAttempt && (clusterState & GPUTPCGMMergedTrackHit::flagReject); if (!retryAttempt) { - if (param.rec.tpc.mergerInterpolateErrors) { - if (iWay == nWays - 2) { - if (!param.rec.tpc.rebuildTrackInFit) { - if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); - } else { - rejectChi2 = true; - } - } else if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { - const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation - candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; - } - } - } else if (iWay == nWays - 1) { - if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { - rejectChi2 = true; + if (finalFit && (param.rec.tpc.mergerInterpolateErrors || param.rec.tpc.rebuildTrackInFit)) { + if (param.rec.tpc.mergerInterpolateRejectAlsoOnCurrentPosition && GetNDF() > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF) { + rejectChi2 = true; + } + } else if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 2) { + if (inter.isValid()) { + retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + } else { + rejectChi2 = true; + } + } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { + if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut + auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; + if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { + const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation + candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; } } - } else { + } else if (!param.rec.tpc.mergerInterpolateErrors && !param.rec.tpc.rebuildTrackInFit) { rejectChi2 = allowChangeClusters; } } From e7c5043498a4e19830c7d1c62aec6d8a67112fcb Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 15 Nov 2026 14:18:02 +0100 Subject: [PATCH 39/54] GPU TPC: Some fixes for NTolerated and nMissed, and some cleanup --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 10 ++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 40 ++++++++++--------- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index c31c12f8c945f..6f5d3f72212ba 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -134,7 +134,9 @@ AddOptionRTC(mergerLooperQPtB5Limit, uint8_t, 4, "", 0, "Min Q/Pt (@B=0.5T) to r AddOptionRTC(mergerLooperSecondHorizontalQPtB5Limit, uint8_t, 2, "", 0, "Min Q/Pt (@B=0.5T) to attempt second horizontal merge between sectors after a vertical merge was found") AddOptionRTC(trackFollowingMaxRowGap, uint8_t, 4, "", 0, "Maximum number of consecutive rows without hit in track following") AddOptionRTC(trackFollowingMaxRowGapSeed, uint8_t, 2, "", 0, "Maximum number of consecutive rows without hit in track following during fit of seed") -AddOptionRTC(trackFitMaxRowMissedHard, uint8_t, 10, "", 0, "Hard limit for number of missed rows in fit / propagation") +AddOptionRTC(trackFitAbortNMissedHits, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") +AddOptionRTC(trackFitAbortNMissedHitsHard, uint8_t, 10, "", 0, "Hard limit for number of missed rows in fit / propagation") +AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(extrapolationTrackingRowRange, uint8_t, 45, "", 0, "Number of rows from the upped/lower limit to search for extrapolated track candidates in for") AddOptionRTC(extrapolationTrackingMinRows, uint8_t, 10, "", 0, "Min num of rows an additional extrapolated track must span over") AddOptionRTC(extrapolationTrackingMinHits, uint8_t, 8, "", 0, "Min num of hits for an additional extrapolated track") @@ -156,8 +158,6 @@ AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely dur AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history -AddOptionRTC(trackFitRejectMode, int8_t, 5, "", 0, "0: no limit on rejection or missed hits, >0: break after n rejected hits") -AddOptionRTC(rejectIFCLowRadiusCluster, uint8_t, 1, "", 0, "Reject clusters that get the IFC mask error during refit") AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") @@ -170,7 +170,7 @@ AddOptionRTC(sigBitsCharge, uint8_t, 4, "", 0, "Number of significant bits for T AddOptionRTC(sigBitsWidth, uint8_t, 3, "", 0, "Number of significant bits for TPC cluster width in compression mode 1") AddOptionRTC(dropLoopers, uint8_t, 0, "", 0, "Drop looping tracks starting from second loop") AddOptionRTC(mergeCE, uint8_t, 1, "", 0, "Merge tracks accross the central electrode") -AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without cluster rejection when fit fails") +AddOptionRTC(retryRefit, int8_t, 1, "", 0, "Retry refit with seeding errors and without chi2 cluster rejection when fit fails") AddOptionRTC(enablePID, int8_t, 1, "", 0, "Enable PID response") AddOptionRTC(PID_useNsigma, int8_t, 1, "", 0, "Use nSigma instead of absolute distance in PID response") AddOptionRTC(adddEdxSubThresholdClusters, int8_t, 1, "", 0, "Add sub threshold clusters in TPC dEdx computation") @@ -499,7 +499,7 @@ AddShortcut("compare", 0, "--QAinput", "Compare QA histograms", "-c", "--qa", "- AddHelp("help", 'h') EndConfig() -//Settings for the O2 workflow +// Settings for the O2 workflow #if !defined(QCONFIG_PARSER_CXX) && !defined(GPUCA_STANDALONE) BeginSubConfig(GPUSettingsO2, global, configStandalone, "O2", 0, "O2 workflow settings", global) AddOption(solenoidBzNominalGPU, float, -1e6f, "", 0, "Field strength of solenoid Bz in kGaus") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 97296e6808cbf..d303d2baf8d54 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -92,8 +92,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? - int32_t nMissed = 0, nMissed2 = 0; - float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! + int32_t nMissed = 0, nMissedHard = 0; // nMissed: cluster rejected, fit failure. nMissedHard: also propagation failure / high incl. + float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; int32_t resetT0 = CAMath::Max(10, CAMath::Min(40, 150.f / CAMath::Abs(mP[4]))); @@ -114,7 +114,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ ConstrainSinPhi(iWay == 0 ? 0.95f : constants::MAX_SIN_PHI_LOW); CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); - N = 0; + N = NTolerated = 0; uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; bool fitBroken = false; float lastUpdateX = -1; @@ -123,14 +123,18 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { - if ((param.rec.tpc.trackFitRejectMode > 0 && nMissed >= param.rec.tpc.trackFitRejectMode) || nMissed2 >= param.rec.tpc.trackFitMaxRowMissedHard || (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { - CADEBUG(printf("\tSkipping hit %d, %d hits rejected, flag %X\n", ihit, nMissed, (int32_t)clusters[ihit].state)); + if ((param.rec.tpc.trackFitAbortNMissedHits > 0 && nMissed >= param.rec.tpc.trackFitAbortNMissedHits) || nMissedHard >= param.rec.tpc.trackFitAbortNMissedHitsHard) { // Too many missed hits in a row, abort this way and drop the remaining hits + CADEBUG(printf("\tAborting at hit %d: %d, %d hits rejected\n", ihit, nMissed, nMissedHard)); + if (finalOutInFit) { + MarkClusters(clusters, ihit, wayDirection > 0 ? maxN - 1 : 0, wayDirection, GPUTPCGMMergedTrackHit::flagRejectErr); + } + break; + } + if ((clusters[ihit].state & GPUTPCGMMergedTrackHit::flagReject) || (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl))) { + CADEBUG(printf("\tSkipping hit %d, flag %X\n", ihit, (int32_t)clusters[ihit].state)); if (rebuilt && (clusters[ihit].state & GPUTPCGMMergedTrackHit::flagHighIncl)) { NTolerated++; } - if (finalOutInFit && !(clusters[ihit].state & (GPUTPCGMMergedTrackHit::flagReject | GPUTPCGMMergedTrackHit::flagHighIncl))) { - clusters[ihit].state |= GPUTPCGMMergedTrackHit::flagRejectErr; - } continue; } } @@ -146,7 +150,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const auto& cluster = clusters[ihit]; CADEBUG(printf("\tSector %2d %11sTrack Alpha %8.3f %s, X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) %28s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", (int32_t)cluster.sector, "", prop.GetAlpha(), (CAMath::Abs(prop.GetAlpha() - clAlpha) < 0.01 ? " " : " R!"), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); - CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed)); + CADEBUG(printf("\tHit %3d/%3d Row %3d: Cluster Alpha %8.3f %3d, X %8.3f - Y %8.3f, Z %8.3f (Missed %d, %d)\n", ihit, maxN, (int32_t)clusters[ihit].row, clAlpha, (int32_t)clusters[ihit].sector, xx, yy, zz, nMissed, nMissedHard)); const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); if (rebuilt) { @@ -197,7 +201,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ retValProp) { // failed for other reason but rotation CADEBUG(printf(" --- break-prop\n")); MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); - nMissed2++; + nMissedHard++; NTolerated++; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); @@ -241,7 +245,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (currentClusterStatus) { nMissed++; - nMissed2++; + nMissedHard++; continue; } @@ -262,7 +266,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ mC[1] = mC[3] = mC[4] = mC[6] = mC[7] = mC[10] = mC[11] = 0.f; lastUpdateX = mX; } else { - nMissed2++; + nMissedHard++; } MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); CADEBUG(printf(" --- break-sinphi\n")); @@ -283,7 +287,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } - int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissed2, uncorrectedY, retryAttempt); + int32_t retValHit = FitHit(merger, iTrk, track, xx, yy, zz, clusterState, clAlpha, iWay, inFlyDirection, deltaZ, lastUpdateX, clusters, prop, inter, dEdx, dEdxAlt, sumInvSqrtCharge, nAvgCharge, ihit, ihitMergeFirst, allowChangeClusters, refit, finalFit, nMissed, nMissedHard, uncorrectedY, retryAttempt); if (retValHit == 0) { DodEdx(dEdx, dEdxAlt, merger, finalFit, ihit, ihitMergeFirst, wayDirection, clusters, clusterState, zz, dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, iTrk)); if (CAMath::Abs(mP[4]) * param.qptB5Scaler > 10 && --resetT0 <= 0 && CAMath::Abs(mP[2]) < 0.15f && CAMath::Square(mP[0] - prop.Model().Y()) + CAMath::Square(mP[1] - prop.Model().Z()) > 1) { // TODO: Could be investigated to reinit the linearization also when sinPhi drifts away from it (a reinit when QPt differs makes things worse, early in the way the QPt of the linearization is usually the better one) @@ -567,7 +571,7 @@ GPUdii() void GPUTPCGMTrackParam::HandleCrossCE(const GPUParam& GPUrestrict() pa } } -GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissed2, float uncorrectedY, bool retryAttempt) +GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& GPUrestrict() deltaZ, float& GPUrestrict() lastUpdateX, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, GPUTPCGMPropagator& GPUrestrict() prop, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, GPUdEdx& GPUrestrict() dEdx, GPUdEdx& GPUrestrict() dEdxAlt, float& GPUrestrict() sumInvSqrtCharge, int32_t& GPUrestrict() nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& GPUrestrict() nMissed, int32_t& GPUrestrict() nMissedHard, float uncorrectedY, bool retryAttempt) { const GPUParam& GPUrestrict() param = merger.Param(); const int32_t nWays = param.rec.tpc.nWays; @@ -636,7 +640,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger if (!retValUpd && !retValInt) // track is updated { lastUpdateX = mX; - nMissed = nMissed2 = 0; + nMissed = nMissedHard = 0; UnmarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); return 0; // ok } else if (retValInt || retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected) { // cluster far away form the track @@ -645,10 +649,8 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } else if (finalFit) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, retValUpd >= GPUTPCGMPropagator::updateErrorClusterRejected ? GPUTPCGMMergedTrackHit::flagRejectDistance : GPUTPCGMMergedTrackHit::flagRejectErr); } - if (!retValInt) { - nMissed++; - nMissed2++; - } + nMissed++; + nMissedHard++; return -1; // cluster rejected } else { return 1; // bad chi2 for the whole track, stop the fit diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 1d98920cfb094..96d37d2fd3fae 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -156,7 +156,7 @@ class GPUTPCGMTrackParam GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); - GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissed2, float uncorrectedY, bool retryAttempt); + GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); From c0f3f468d3689141b2d64e8823fc11ad2165e0dd Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 16 Nov 2026 22:51:44 +0100 Subject: [PATCH 40/54] GPU QA: Improve Efficiency Stat Printing --- GPU/GPUTracking/qa/GPUQA.cxx | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 2038c4c84f707..fd0fb18b6dfa3 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2355,10 +2355,31 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } if (mConfig.enableLocalOutput && !mConfig.inputHistogramsOnly && (mQATasks & taskTrackingEff) && mcPresent()) { - GPUInfo("QA Stats: Eff: Tracks Prim %d (Eta %d, Pt %d) %f%% (%f%%) Sec %d (Eta %d, Pt %d) %f%% (%f%%) - Res: Tracks %d (Eta %d, Pt %d)", (int32_t)mEff[3][1][0][0]->GetEntries(), (int32_t)mEff[3][1][0][3]->GetEntries(), (int32_t)mEff[3][1][0][4]->GetEntries(), - mEff[0][0][0][0]->GetSumOfWeights() / std::max(1., mEff[3][0][0][0]->GetSumOfWeights()), mEff[0][1][0][0]->GetSumOfWeights() / std::max(1., mEff[3][1][0][0]->GetSumOfWeights()), (int32_t)mEff[3][1][1][0]->GetEntries(), (int32_t)mEff[3][1][1][3]->GetEntries(), - (int32_t)mEff[3][1][1][4]->GetEntries(), mEff[0][0][1][0]->GetSumOfWeights() / std::max(1., mEff[3][0][1][0]->GetSumOfWeights()), mEff[0][1][1][0]->GetSumOfWeights() / std::max(1., mEff[3][1][1][0]->GetSumOfWeights()), (int32_t)mRes2[0][0]->GetEntries(), - (int32_t)mRes2[0][3]->GetEntries(), (int32_t)mRes2[0][4]->GetEntries()); + GPUInfo("QA Stats: Tracking performance vs Pt: average over the Pt bins (all tracks)"); + static constexpr const char* names[4] = {"Prim findable", "Prim all", "Sec findable", "Sec all"}; + for (int32_t j = 0; j < 4; j++) { + const int32_t iFindable = j % 2, iSec = j / 2; // index 0 of the findable dimension are the findable tracks, index 1 all tracks + double sumNum[3] = {}, sumDen[3] = {}, sumRatio[3] = {}; + int32_t nBins[3] = {}; + for (int32_t bin = 1; bin <= mEff[0][iFindable][iSec][4]->GetNbinsX(); bin++) { + const double rec = mEff[0][iFindable][iSec][4]->GetBinContent(bin), clone = mEff[1][iFindable][iSec][4]->GetBinContent(bin), fake = mEff[2][iFindable][iSec][4]->GetBinContent(bin); + const double num[3] = {rec, clone, fake}, den[3] = {mEff[5][iFindable][iSec][4]->GetBinContent(bin), rec + clone, rec + clone + fake}; // Same ratios as in the plots + for (int32_t l = 0; l < 3; l++) { + sumNum[l] += num[l]; + sumDen[l] += den[l]; + if (den[l] > 0.) { + sumRatio[l] += num[l] / den[l]; + nBins[l]++; + } + } + } + double avg[3], total[3]; + for (int32_t l = 0; l < 3; l++) { + avg[l] = nBins[l] ? 100. * sumRatio[l] / nBins[l] : 0.; + total[l] = sumDen[l] > 0. ? 100. * sumNum[l] / sumDen[l] : 0.; + } + GPUInfo("QA Stats: %-13s - Eff %5.2f (%6.2f) - Clone %5.2f (%6.2f) - Fake %5.2f (%6.2f)", names[j], avg[0], total[0], avg[1], total[1], avg[2], total[2]); + } } int32_t flagShowVsPtLog = (mConfig.enableLocalOutput || mConfig.shipToQCAsCanvas) ? 1 : 0; From 23838ada3ef73b8c788c0be91d329535f33de5b5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 16 Nov 2026 20:12:25 +0100 Subject: [PATCH 41/54] GPU TPC: Make AttachClusters work with track rebuilding --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 2 +- GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx | 64 ----- GPU/GPUTracking/Merger/GPUTPCGMPropagator.h | 1 - GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 218 +++++++++++++----- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 16 +- 5 files changed, 165 insertions(+), 136 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 6f5d3f72212ba..10095eb69ec2c 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -157,7 +157,7 @@ AddOptionRTC(mergerNonInterpolateRejectMinNDF, uint8_t, 5, "", 0, "Minimum NDF o AddOptionRTC(rebuildTrackInFit, uint8_t, 1, "", 0, "Rebuild track completely during fit based on clusters closed to interpolated track positions, adds 2 fit iterations") AddOptionRTC(rebuildTrackInFitClusterCandidates, uint8_t, 3, "", 0, "Number of cluster candidates per row for rebuilt track") AddOptionRTC(disableRebuildAttachment, uint8_t, 0, "", 0, "Bitmask to disable certain attachment steps during track rebuid (1: current row, 2: interpolate missing row, 4: one sided interpolation, 8: original hit, 16: extrapolation)") -AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: extrapolate missing rows, 4: loop following)") // TODO: Add history +AddOptionRTC(disableMarkAdjacent, uint8_t, 0, "", 0, "Bitmask to disable certain steps during refit to mark adjacenrt clusters (1: current row, 2: interpolate missing rows, 4: extrapolate inwards / outwards, 8: loop following)") // TODO: Add history AddOptionRTC(dEdxTruncLow, uint8_t, 2, "", 0, "Low truncation threshold, fraction of 128") AddOptionRTC(dEdxTruncHigh, uint8_t, 77, "", 0, "High truncation threshold, fraction of 128") AddOptionRTC(extrapolationTracking, int8_t, 1, "", 0, "Enable Extrapolation Tracking (prolong tracks to adjacent sectors to find short segments)") diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx index ddfefe8636af5..fcaa430bfdd94 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.cxx @@ -652,70 +652,6 @@ GPUd() void GPUTPCGMPropagator::InterpolateFill(gputpcgmmergertypes::Interpolati } } -GPUd() int32_t GPUTPCGMPropagator::InterpolateReject(const GPUParam& GPUrestrict() param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ) -{ - float* GPUrestrict() mC = mT->Cov(); - float* GPUrestrict() mP = mT->Par(); - float chi2Y, chi2Z; - if (mT->NDF() <= 0) { - chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); - chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); - } else if (mFitInProjections) { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); - const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); - const float Ik00 = mC[0] * Iw0; - const float Ik11 = mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0; - const float ImP1 = mP[1] + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0]; - const float ImC2 = mC[2] - Ik11 * mC[2]; - // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - const float Jw0 = 1.f / (ImC0 + err2Y); - const float Jw2 = 1.f / (ImC2 + err2Z); - chi2Y = Jw0 * Jz0 * Jz0; - chi2Z = Jw2 * Jz1 * Jz1; - } else { - const float Iz0 = inter->posY - mP[0]; - const float Iz1 = inter->posZ + deltaZ - mP[1]; - float Iw0 = mC[2] + (float)inter->errorZ; - float Iw2 = mC[0] + (float)inter->errorY; - float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); - Idet = 1.f / Idet; - Iw0 *= Idet; - const float Iw1 = mC[1] * Idet; - Iw2 *= Idet; - const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; - const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; - const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; - const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; - const float ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; - const float ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; - const float ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; - const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; - const float ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; - const float Jz0 = posY - ImP0; - const float Jz1 = posZ - ImP1; - float Jw0 = ImC2 + err2Z; - float Jw2 = ImC0 + err2Y; - float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); - Jdet = 1.f / Jdet; - Jw0 *= Jdet; - const float Jw1 = ImC1 * Jdet; - Jw2 *= Jdet; - chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); - chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); - } - if (RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? - // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); - return updateErrorClusterRejectedInInterpolation; - } - return 0; -} - GPUd() int32_t GPUTPCGMPropagator::Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* GPUrestrict() param) { float* GPUrestrict() mC = mT->Cov(); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h index 3f1a74cfb6c26..9c31069da5302 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMPropagator.h @@ -103,7 +103,6 @@ class GPUTPCGMPropagator GPUd() int32_t Update(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, bool rejectChi2, bool refit, float err2Y, float err2Z); GPUd() int32_t Update(float posY, float posZ, int16_t clusterState, bool rejectChi2, float err2Y, float err2Z, const GPUParam* param = nullptr); GPUd() void InterpolateFill(gputpcgmmergertypes::InterpolationErrorHit* inter); - GPUd() int32_t InterpolateReject(const GPUParam& param, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ); GPUd() float PredictChi2(float posY, float posZ, int32_t iRow, const GPUParam& param, int16_t clusterState, uint8_t sector, float time, float avgCharge, float charge) const; GPUd() float PredictChi2(float posY, float posZ, float err2Y, float err2Z) const; GPUd() static int32_t RejectCluster(float chiY, float chiZ, uint8_t clusterState) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index d303d2baf8d54..98593716d99c2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -90,6 +90,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float covYYUpd = 0.f; float deltaZ = 0.f; const int32_t wayRebuildStart = param.rec.tpc.mergerInitial3wayFit ? 2 : 0; + const bool adjacentInRebuild = param.rec.tpc.rebuildTrackInFit && !param.rec.tpc.mergerInterpolateErrors; + const int8_t adjacentLeg = 1 + (track.Leg() == 0); for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? int32_t nMissed = 0, nMissedHard = 0; // nMissed: cluster rejected, fit failure. nMissedHard: also propagation failure / high incl. @@ -115,13 +117,18 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("Fitting track %d way %d (sector %d, alpha %f) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n", iTrk, iWay, CAMath::Float2IntRn(prop.GetAlpha() / GPUTPCGeometry::kSectAngle()) + (mP[1] < 0 ? 18 : 0), prop.GetAlpha())); N = NTolerated = 0; - uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255; + uint8_t lastUpdateRow = 255, lastPropagateRow = 255, lastSector = 255, lastIndexRow = 255, lastIndexSector = 255; bool fitBroken = false; float lastUpdateX = -1; const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; - for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection, interpolationIndex += wayDirection) { + for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { + if (lastIndexRow != 255) { // One index per pad row (2 if the track has clusters of 2 sectors in the row), counted also for hits skipped below, such that all ways use the same index for a hit + interpolationIndex += (clusters[ihit].row == lastIndexRow ? (int32_t)(clusters[ihit].sector != lastIndexSector) : CAMath::Abs(clusters[ihit].row - lastIndexRow)) * wayDirection; + } + lastIndexRow = clusters[ihit].row; + lastIndexSector = clusters[ihit].sector; if (!param.rec.tpc.rebuildTrackInFit || rebuilt) { if ((param.rec.tpc.trackFitAbortNMissedHits > 0 && nMissed >= param.rec.tpc.trackFitAbortNMissedHits) || nMissedHard >= param.rec.tpc.trackFitAbortNMissedHitsHard) { // Too many missed hits in a row, abort this way and drop the remaining hits CADEBUG(printf("\tAborting at hit %d: %d, %d hits rejected\n", ihit, nMissed, nMissedHard)); @@ -155,8 +162,8 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int32_t rowGap = lastPropagateRow == 255 ? 0 : CAMath::Max(0, CAMath::Abs(cluster.row - lastPropagateRow) - 1); if (rebuilt) { interpolationIndex = ihit; - } else if ((uint32_t)(interpolationIndex + rowGap * wayDirection) >= interpolation.size) { // covers all indices of the row gap, since they are monotonic - merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex + rowGap * wayDirection, interpolation.size); + } else if ((uint32_t)interpolationIndex >= interpolation.size) { // covers all indices of the row gap, since they are monotonic + merger.raiseError(GPUErrors::ERROR_MERGER_INTERPOLATION_OVERFLOW, interpolationIndex, interpolation.size); fitBroken = true; break; } @@ -164,15 +171,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ uint8_t dEdxSubThresholdRow = 255; if (rowGap) { const bool dodEdx = param.dodEdxEnabled && param.rec.tpc.adddEdxSubThresholdClusters && finalFit && CAMath::Abs(cluster.row - lastUpdateRow) == 2 && cluster.sector == lastSector && currentClusterStatus == 0; - const bool doAttach = allowChangeClusters && !param.rec.tpc.rebuildTrackInFit && !(merger.Param().rec.tpc.disableMarkAdjacent & 2); const bool crossCE = lastSector != 255 && ((cluster.sector >= GPUTPCGeometry::NSECTORS / 2) != (lastSector >= GPUTPCGeometry::NSECTORS / 2)); const bool ceOverlap = crossCE && ((cluster.row > lastPropagateRow) != ((clusters[0].row < clusters[maxN - 1].row) ^ (iWay & 1))); // Row gap goes backwards at the CE crossing, these rows were already searched on both sides + const bool doAttach = (adjacentInRebuild ? (iWay == wayRebuildStart + 1 && !ceOverlap) : (allowChangeClusters && !param.rec.tpc.rebuildTrackInFit)) && !(param.rec.tpc.disableMarkAdjacent & 2); const uint8_t doInterpolate = (!param.rec.tpc.rebuildTrackInFit || (param.rec.tpc.disableRebuildAttachment & 2) || ceOverlap) ? 0 : (iWay == wayRebuildStart ? 1 : (iWay == wayRebuildStart + 1 ? 2 : 0)); if (lastUpdateRow != 255 && (dodEdx || doAttach || doInterpolate)) { int32_t step = cluster.row > lastPropagateRow ? 1 : -1; uint8_t sector = lastSector; - for (int32_t iRow = lastPropagateRow + step, index = interpolationIndex; iRow != cluster.row; iRow += step, index += wayDirection) { + for (int32_t iRow = lastPropagateRow + step, index = rebuilt ? interpolationIndex : (interpolationIndex - rowGap * wayDirection); iRow != cluster.row; iRow += step, index += wayDirection) { float tmpX, tmpY, tmpZ; if (prop.GetPropagatedYZ(GPUTPCGeometry::Row2X(iRow), tmpY, tmpZ)) { break; @@ -187,9 +194,6 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } } - if (!rebuilt) { - interpolationIndex += rowGap * wayDirection; - } } lastPropagateRow = cluster.row; @@ -204,7 +208,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ nMissedHard++; NTolerated++; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } continue; } @@ -214,22 +218,17 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ // clang-format on if (mNDF >= 0 && (mC[0] > param.rec.tpc.trackFitCovLimit || mC[2] > param.rec.tpc.trackFitCovLimit)) { if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } fitBroken = true; break; // bad chi2 for the whole track, stop the fit } float uncorrectedY = -1e6f; - if (param.rec.tpc.rebuildTrackInFit) { - if (iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); // TODO: Check whether we do not need to use track Y for some rejection steps - } - if (allowChangeClusters) { - AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); // TODO: Do this during FindBestInterpolatedHit - } - } else if (allowChangeClusters) { - uncorrectedY = AttachClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); + if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps + } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isValid()) : allowChangeClusters)) { + uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { float tmpZ; @@ -308,7 +307,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } } else if (retValHit == 1) { if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { - InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk); + InterpolateMissingRows(merger, interpolation, clusters, ihit + wayDirection, interpolationIndex + wayDirection, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } fitBroken = true; break; @@ -381,7 +380,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Propaga Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f) --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10])); gputpcgmmergertypes::InterpolationErrorHit inter; inter.markInvalid(); - float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, true); + float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; bool found = candidate.id >= 2; if (found) { @@ -468,7 +467,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ candidate.best = -1; } } - if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped + if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped StoreLoopPropagation(merger, sector, iExtrapolatedRow, iTrk, up == 1, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d extrapolated row %d sector %d out %d\n", iTrk, (int)iExtrapolatedRow, sector, up == 1)); } @@ -477,7 +476,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ *this = paramStart; prop.SetTrack(this, alphaStart); } - } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 4) && lastUpdateRow != 255 && !retryAttempt) { + } else if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && lastUpdateRow != 255 && !retryAttempt) { StoreLoopPropagation(merger, lastSector, lastUpdateRow, iTrk, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row, prop.GetAlpha()); CADEBUG(printf("\t\tSTORING %d lastUpdateRow %d row %d out %d\n", iTrk, (int)lastUpdateRow, (int)clusters[(iWay & 1) ? (maxN - 1) : 0].row, lastUpdateRow > clusters[(iWay & 1) ? (maxN - 1) : 0].row)); } @@ -540,18 +539,18 @@ GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t se } } } - if (doAttach) { - AttachClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); - } - if (doInterpolate) { - if (doInterpolate == 2) { - if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { - inter.markInvalid(); - } - FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false); - } else { + if (doInterpolate == 2) { + if (inter.isValid() && CAMath::Abs(inter.posY - mP[0]) > param.rec.tpc.rebuildTrackMaxInterpolationDistY) { + inter.markInvalid(); + } + FindBestInterpolatedHit(merger, inter, sector, iRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, doAttach ? (1 + (track.Leg() == 0)) : 0); + } else { + if (doInterpolate == 1) { prop.InterpolateFill(&inter); } + if (doAttach) { + FindAdjacentClusters(merger, sector, iRow, iTrk, track.Leg() == 0, prop); + } } } @@ -605,14 +604,14 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } else if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 2) { if (inter.isValid()) { - retValInt = prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ); + retValInt = InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, (param.rec.tpc.disableMarkAdjacent & 1) ? 0 : (1 + (track.Leg() == 0))); } else { rejectChi2 = true; } } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidate.id < 2 && !prop.InterpolateReject(param, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ)) { + if (candidate.id < 2 && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; } @@ -657,7 +656,82 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } } -GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, bool extrapolation) +GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUrestrict() merger, const GPUTPCGMPropagator& GPUrestrict() prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach) +{ + const GPUParam& GPUrestrict() param = merger.Param(); + float chi2Y, chi2Z, ImP0, ImP1, ImC0, ImC2; // Interpolated position and errors + if (mNDF <= 0) { + ImP0 = (float)inter->posY; + ImP1 = (float)inter->posZ + deltaZ; + ImC0 = (float)inter->errorY; + ImC2 = (float)inter->errorZ; + chi2Y = CAMath::Square((float)inter->posY - posY) / ((float)inter->errorY + err2Y); + chi2Z = CAMath::Square((float)inter->posZ + deltaZ - posZ) / ((float)inter->errorZ + err2Z); + } else if (prop.GetFitInProjections()) { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + const float Iw0 = 1.f / (mC[0] + (float)inter->errorY); + const float Iw2 = 1.f / (mC[2] + (float)inter->errorZ); + const float Ik00 = mC[0] * Iw0; + const float Ik11 = mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0; + ImP1 = mP[1] + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0]; + ImC2 = mC[2] - Ik11 * mC[2]; + // printf("\t%21sInterpo ----- abde artaf%16s Y %8.3f, Z %8.3f (Errors %f <-- (%f, %f) %f <-- (%f, %f))\n", "", "", ImP0, ImP1, sqrtf(ImC0), sqrtf(mC[0]), sqrtf(inter->errorY), sqrtf(ImC2), sqrtf(mC[2]), sqrtf(inter->errorZ)); + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + const float Jw0 = 1.f / (ImC0 + err2Y); + const float Jw2 = 1.f / (ImC2 + err2Z); + chi2Y = Jw0 * Jz0 * Jz0; + chi2Z = Jw2 * Jz1 * Jz1; + } else { + const float Iz0 = inter->posY - mP[0]; + const float Iz1 = inter->posZ + deltaZ - mP[1]; + float Iw0 = mC[2] + (float)inter->errorZ; + float Iw2 = mC[0] + (float)inter->errorY; + float Idet = CAMath::Max(1e-10f, Iw0 * Iw2 - mC[1] * mC[1]); + Idet = 1.f / Idet; + Iw0 *= Idet; + const float Iw1 = mC[1] * Idet; + Iw2 *= Idet; + const float Ik00 = mC[0] * Iw0 + mC[1] * Iw1; + const float Ik01 = mC[0] * Iw1 + mC[1] * Iw2; + const float Ik10 = mC[1] * Iw0 + mC[2] * Iw1; + const float Ik11 = mC[1] * Iw1 + mC[2] * Iw2; + ImP0 = mP[0] + Ik00 * Iz0 + Ik01 * Iz1; + ImP1 = mP[1] + Ik10 * Iz0 + Ik11 * Iz1; + ImC0 = mC[0] - Ik00 * mC[0] + Ik01 * mC[1]; + const float ImC1 = mC[1] - Ik10 * mC[0] + Ik11 * mC[1]; + ImC2 = mC[2] - Ik10 * mC[1] + Ik11 * mC[2]; + const float Jz0 = posY - ImP0; + const float Jz1 = posZ - ImP1; + float Jw0 = ImC2 + err2Z; + float Jw2 = ImC0 + err2Y; + float Jdet = CAMath::Max(1e-10f, Jw0 * Jw2 - ImC1 * ImC1); + Jdet = 1.f / Jdet; + Jw0 *= Jdet; + const float Jw1 = ImC1 * Jdet; + Jw2 *= Jdet; + chi2Y = CAMath::Abs((Jw0 * Jz0 + Jw1 * Jz1) * Jz0); + chi2Z = CAMath::Abs((Jw1 * Jz0 + Jw2 * Jz1) * Jz1); + } + if (doAttach) { // Search adjacent clusters here, at the interpolated position with its errors (1 = other leg, 2 = good leg) + float uncorrectedY, uncorrectedZ, clErr2Y, clErr2Z; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, ImP0, ImP1, uncorrectedY, uncorrectedZ); + if (CAMath::Abs(uncorrectedY) <= merger.GetConstantMem()->tpcTrackers[sector].Row(row).getTPCMaxY()) { + param.GetClusterErrors2(sector, row, ImP1, mP[2], mP[3], -1.f, 0.f, 0.f, clErr2Y, clErr2Z); // TODO: Use correct time/avgCharge + FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, clErr2Y + CAMath::Abs(ImC0), clErr2Z + CAMath::Abs(ImC2)); + } + } + if (GPUTPCGMPropagator::RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? + // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); + return GPUTPCGMPropagator::updateErrorClusterRejectedInInterpolation; + } + return 0; +} + +GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation) { const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); @@ -671,7 +745,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre useInter = (float)inter.errorY < maxNonIntCov && (float)inter.errorZ < maxNonIntCov; } const bool useCurrent = !interOnly && maxNonIntCov > 0 && mC[0] < maxNonIntCov && mC[2] < maxNonIntCov && mNDF > (int32_t)param.rec.tpc.rebuildTrackNonIntMinNDF; - if (rowData.NHits() && (useInter || useCurrent)) { + if (useInter || useCurrent) { const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; const float y0 = rowData.Grid().YMin(); const float stepY = rowData.HstepY(); @@ -726,6 +800,9 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { float err2Y, err2Z; param.GetClusterErrors2(sector, row, zMoved, sinPhiMoved, mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + if (doAttach) { + FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, err2Y + CAMath::Abs(ImC0), err2Z + CAMath::Abs(ImC2)); + } const float kFactor = tracker.GetChiSeedFactor(); const float sy2 = 4 * CAMath::Min(param.rec.tpc.hitSearchArea2, kFactor * (err2Y + CAMath::Abs(ImC0))); // TODO: is 4 a good factor?? @@ -794,11 +871,19 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; } + } else if (doAttach && (!interOnly || inter.isValid())) { + const float y = interOnly ? (float)inter.posY : mP[0], z = interOnly ? (float)inter.posZ + deltaZ : mP[1]; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, row, y, z, uncorrectedY, uncorrectedZ); + if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { + float err2Y, err2Z; + param.GetClusterErrors2(sector, row, z, mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, z, err2Y + (interOnly ? (float)inter.errorY : CAMath::Abs(mC[0])), err2Z + (interOnly ? (float)inter.errorZ : CAMath::Abs(mC[2]))); + } } return uncorrectedY; } -GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk) +GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrestrict() merger, gputpcgmmergertypes::InterpolationErrors& GPUrestrict() interpolation, GPUTPCGMMergedTrackHit* GPUrestrict() clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& GPUrestrict() prop, const int32_t iTrk, const int8_t doAttach) { for (; ihit >= 0; ihit--, interpolationIndex--) { while (ihit > 0 && clusters[ihit].row == clusters[ihit - 1].row && clusters[ihit].sector == clusters[ihit - 1].sector) { @@ -813,7 +898,7 @@ GPUdni() void GPUTPCGMTrackParam::InterpolateMissingRows(GPUTPCGMMerger& GPUrest return; } auto inter = interpolation.hit[interpolationIndex]; - FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true); + FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, doAttach); } } @@ -998,7 +1083,7 @@ GPUd() int32_t GPUTPCGMTrackParam::MergeDoubleRowClusters(int32_t& ihit, int32_t return 0; } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) +GPUd() float GPUTPCGMTrackParam::FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop) { float Y, Z; float X = 0; @@ -1007,22 +1092,38 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric Y = mP[0]; Z = mP[1]; } - return AttachClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); + return FindAdjacentClusters(merger, sector, iRow, iTrack, goodLeg, Y, Z); } -GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) +GPUd() float GPUTPCGMTrackParam::FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z) { const GPUParam& GPUrestrict() param = merger.Param(); + const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); + const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); if (param.rec.tpc.disableMarkAdjacent & 1) { return -1e6f; } + + float uncorrectedY, uncorrectedZ; + merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); + if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { + return uncorrectedY; + } + + float err2Y, err2Z; + param.GetClusterErrors2(sector, iRow, Z, mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge + FindAdjacentClustersUncorrectedPos(merger, sector, iRow, iTrack, goodLeg, uncorrectedY, uncorrectedZ, Z, err2Y + CAMath::Abs(mC[0]), err2Z + CAMath::Abs(mC[2])); // Cov can be bogus when following circle + + return uncorrectedY; +} + +GPUd() void GPUTPCGMTrackParam::FindAdjacentClustersUncorrectedPos(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float uncorrectedY, float uncorrectedZ, float Z, float err2Y, float err2Z) +{ + const GPUParam& GPUrestrict() param = merger.Param(); const GPUTPCTracker& GPUrestrict() tracker = *(merger.GetConstantMem()->tpcTrackers + sector); const GPUTPCRow& GPUrestrict() row = tracker.Row(iRow); GPUglobalref() const cahit2* hits = tracker.HitData(row); GPUglobalref() const calink* firsthit = tracker.FirstHitInBin(row); - if (row.NHits() == 0) { - return -1e6f; - } const float zOffset = param.par.continuousTracking ? merger.GetConstantMem()->calibObjects.fastTransform->convVertexTimeToZOffset(sector, mTOffset, param.continuousMaxTimeBin) : 0; // TODO: do some validatiomns for the transform conv functions... const float y0 = row.Grid().YMin(); @@ -1031,15 +1132,7 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric const float stepZ = row.HstepZ(); int32_t bin, ny, nz; - float uncorrectedY, uncorrectedZ; - merger.GetConstantMem()->calibObjects.fastTransform->InverseTransformYZtoNominalYZ(sector, iRow, Y, Z, uncorrectedY, uncorrectedZ); - if (CAMath::Abs(uncorrectedY) > row.getTPCMaxY()) { - return uncorrectedY; - } - bool protect = CAMath::Abs(GetQPt() * param.qptB5Scaler) <= param.rec.tpc.rejectQPtB5 && goodLeg; - float err2Y, err2Z; - param.GetClusterErrors2(sector, iRow, Z, mP[2], mP[3], -1.f, 0.f, 0.f, err2Y, err2Z); // TODO: Use correct time/avgCharge const float tubeMaxSize2 = protect ? param.rec.tpc.tubeProtectMaxSize2 : param.rec.tpc.tubeRemoveMaxSize2; const float tubeMinSize2 = protect ? param.rec.tpc.tubeProtectMinSize2 : 0.f; float tubeSigma2 = protect ? param.rec.tpc.tubeProtectSigma2 : param.rec.tpc.tubeRemoveSigma2; @@ -1050,8 +1143,8 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric param.GetUnscaledMult(time) / GPUTPCGeometry::Row2X(iRow) > param.rec.tpc.tubeExtraProtectMinOccupancy) { tubeSigma2 *= protect ? 2 : 0.5; } - const float sy2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * (err2Y + CAMath::Abs(mC[0])))); // Cov can be bogus when following circle - const float sz2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * (err2Z + CAMath::Abs(mC[2])))); // In that case we should provide the track error externally + const float sy2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * err2Y)); + const float sz2 = CAMath::Max(tubeMinSize2, CAMath::Min(tubeMaxSize2, tubeSigma2 * err2Z)); const float tubeY = CAMath::Sqrt(sy2); const float tubeZ = CAMath::Sqrt(sz2); const float sy21 = 1.f / sy2; @@ -1093,7 +1186,6 @@ GPUd() float GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger& GPUrestric } } } - return uncorrectedY; } GPUdii() void GPUTPCGMTrackParam::StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha) @@ -1157,13 +1249,13 @@ GPUdii() void GPUTPCGMTrackParam::PropagateLooper(const GPUTPCGMMerger& GPUrestr prop.SetTrack(&data.param, data.alpha); if (merger.Param().rec.tpc.looperFollowMode == 1) { - data.param.AttachClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); + data.param.FindAdjacentClustersLooperFollow(merger, prop, data.sector, data.track, data.outwards); } else { - data.param.AttachClustersLooper(merger, data.sector, data.row, data.track, data.outwards, prop); + data.param.FindAdjacentClustersLooperMirror(merger, data.sector, data.row, data.track, data.outwards, prop); } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) +GPUdi() void GPUTPCGMTrackParam::FindAdjacentClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& GPUrestrict() prop, int32_t sector, const int32_t iTrack, const bool up) { float toX = mX; bool inFlyDirection = (merger.MergedTracks()[iTrack].Leg() & 1) ^ up; @@ -1205,7 +1297,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger float rowX = GPUTPCGeometry::Row2X(j); if (CAMath::Abs(rowX - (-mP[0] * lrFactor)) < 1.5f) { CADEBUG(printf("\t\tAttempt row %d (X %f Y %f Z %f)\n", j, rowX, mX * lrFactor, mP[1])); - AttachClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); + FindAdjacentClusters(merger, sector, j, iTrack, false, mX * lrFactor, mP[1]); } } } @@ -1231,7 +1323,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooperFollow(const GPUTPCGMMerger } } -GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) +GPUdi() void GPUTPCGMTrackParam::FindAdjacentClustersLooperMirror(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& GPUrestrict() prop) { // Note that the coordinate system is rotated by 90 degree swapping X and Y! float X = mP[2] > 0 ? mP[0] : -mP[0]; @@ -1280,7 +1372,7 @@ GPUdi() void GPUTPCGMTrackParam::AttachClustersLooper(const GPUTPCGMMerger& GPUr float rowX = mX + GPUTPCGeometry::Row2X(j) - myRowX; if (CAMath::Abs(rowX - paramX) < 1.5f) { // printf("Attempt row %d at y %f\n", j, X); - AttachClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); + FindAdjacentClusters(merger, sector, j, iTrack, false, mP[2] > 0 ? X : -X, Z); } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 96d37d2fd3fae..338f3d7d8b910 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,13 +163,15 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, bool extrapolation = false); - GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk); - - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use - GPUd() float AttachClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); - GPUd() void AttachClustersLooper(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); - GPUd() void AttachClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); + GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg) + GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation = false); + GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, const int8_t doAttach); + + GPUd() float FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, GPUTPCGMPropagator& prop); // Returns uncorrectedY for later use + GPUd() float FindAdjacentClusters(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float Y, float Z); + GPUd() void FindAdjacentClustersUncorrectedPos(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool goodLeg, float uncorrectedY, float uncorrectedZ, float Z, float err2Y, float err2Z); // err2Y / err2Z: cluster error + track error + GPUd() void FindAdjacentClustersLooperMirror(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, const int32_t iTrack, const bool up, const GPUTPCGMPropagator& prop); + GPUd() void FindAdjacentClustersLooperFollow(const GPUTPCGMMerger& GPUrestrict() merger, GPUTPCGMPropagator& prop, int32_t sector, const int32_t iTrack, const bool up); GPUd() void StoreLoopPropagation(const GPUTPCGMMerger& GPUrestrict() merger, int32_t sector, int32_t iRow, int32_t iTrack, bool outwards, float alpha); GPUd() void StoreOuter(gputpcgmmergertypes::GPUTPCOuterParam* outerParam, float alpha); GPUd() static void PropagateLooper(const GPUTPCGMMerger& merger, int32_t loopIdx); From b2e3ab96f9ac4ddefd886ed4d0696810585c7256 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 11:38:46 +0100 Subject: [PATCH 42/54] GPU TPC: Mark hits with findAdjacentClusters done in previous iteration to cover exactly the remaining in last fit iteration. --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 2 ++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 3 ++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 88388a4b9074f..ba6abedbcc1ad 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -37,6 +37,8 @@ struct InterpolationErrorHit { GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } + GPUdi() bool isAdjacentDone() const { return errorY == (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } + GPUdi() void markAdjacentDone() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } // Adjacent clusters were searched at the interpolated position of this entry, the entry is invalid afterwards }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 98593716d99c2..2a89e3814f824 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -227,7 +227,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps - } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isValid()) : allowChangeClusters)) { + } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isAdjacentDone()) : allowChangeClusters)) { uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { @@ -723,6 +723,7 @@ GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUre param.GetClusterErrors2(sector, row, ImP1, mP[2], mP[3], -1.f, 0.f, 0.f, clErr2Y, clErr2Z); // TODO: Use correct time/avgCharge FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, clErr2Y + CAMath::Abs(ImC0), clErr2Z + CAMath::Abs(ImC2)); } + inter->markAdjacentDone(); } if (GPUTPCGMPropagator::RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 338f3d7d8b910..2e8ee86ca8196 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -163,7 +163,7 @@ class GPUTPCGMTrackParam GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); - GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg) + GPUd() int32_t InterpolateReject(const GPUTPCGMMerger& merger, const GPUTPCGMPropagator& prop, float posY, float posZ, int16_t clusterState, gputpcgmmergertypes::InterpolationErrorHit* inter, float err2Y, float err2Z, float deltaZ, uint8_t sector, uint8_t row, int32_t iTrk, int8_t doAttach); // doAttach: search adjacent clusters at the interpolated position (1 / 2 = good leg), marks inter as done GPUd() float FindBestInterpolatedHit(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrorHit& inter, const uint8_t sector, const uint8_t row, const float deltaZ, const float sumInvSqrtCharge, const int nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, bool interOnly, int8_t doAttach, bool extrapolation = false); GPUd() void InterpolateMissingRows(GPUTPCGMMerger& merger, gputpcgmmergertypes::InterpolationErrors& interpolation, GPUTPCGMMergedTrackHit* clusters, int32_t ihit, int32_t interpolationIndex, int32_t lastRow, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge, const GPUTPCGMPropagator& prop, const int32_t iTrk, const int8_t doAttach); From 859770d713ebcd50a650c73eaedfa19e54425eae Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 11:33:18 +0100 Subject: [PATCH 43/54] GPU TPC: Fix cluster adjacent flags with Rebuild (and improve without) --- .../GPUTPCCompressionKernels.cxx | 4 ++-- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 20 +++++++++---------- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 4 ++-- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 9 ++++++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 2 +- .../display/render/GPUDisplayDraw.cxx | 2 +- 6 files changed, 21 insertions(+), 20 deletions(-) diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index 9bc6f40a052fa..67dacf993f0a4 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -53,7 +53,7 @@ GPUdii() void GPUTPCCompressionKernels::Thread= GPUSettings::RejectionStrategyA && !(attach & gputpcgmmergertypes::attachProtect) && (rejectTrk || GPUTPCClusterRejection::GetIsRejected(attach)); @@ -206,7 +206,7 @@ GPUd() bool GPUTPCCompression::rejectCluster(int32_t idx, const GPUParam& GPUres if (attach & gputpcgmmergertypes::attachProtect) { return false; } - int32_t id = attach & gputpcgmmergertypes::attachTrackMask; + int32_t id = gputpcgmmergertypes::getAttachedTrackId(attach); auto& trk = ioPtrs.mergedTracks[id]; if (GPUTPCClusterRejection::IsTrackRejected(trk, param)) { return true; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index f0183cd5472b6..31a6b9a18a10b 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -1878,13 +1878,10 @@ GPUd() void GPUTPCGMMerger::PrepareForFit1(int32_t nBlocks, int32_t nThreads, in mTrackOrderAttach[mTrackSort[i]] = i; GPUTPCGMMergedTrack& trk = mMergedTracks[i]; if (trk.OK()) { + const bool protect = CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0; for (uint32_t j = 0; j < trk.NClusters(); j++) { - if (!Param().rec.tpc.rebuildTrackInFit) { - uint32_t weight = attachAttached | attachGood; - if (CAMath::Abs(trk.GetParam().GetQPt() * Param().qptB5Scaler) <= Param().rec.tpc.rejectQPtB5 && !trk.MergedLooper() && trk.Leg() == 0) { - weight |= attachProtect; - } - CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], weight); + if (protect) { // Protect the clusters of the initial tracks. No track id on purpose, any entry of a final track for the cluster shall win + CAMath::AtomicMax(&mClusterAttachment[mClusters[trk.FirstClusterRef() + j].num], (uint32_t)(attachAttached | attachProtect | (Param().rec.tpc.rebuildTrackInFit ? attachZero : attachGood))); } CAMath::AtomicAdd(&mSharedCount[mClusters[trk.FirstClusterRef() + j].num], 1u); } @@ -1933,8 +1930,8 @@ GPUd() void GPUTPCGMMerger::Finalize0(int32_t nBlocks, int32_t nThreads, int32_t mTrackSort[mTrackOrderAttach[i]] = i; } for (uint32_t i = iBlock * nThreads + iThread; i < mMemory->nMergedTrackClusters; i += nThreads * nBlocks) { - if (!(mClusterAttachment[mClusters[i].num] & attachProtect)) { - mClusterAttachment[mClusters[i].num] = 0; // Reset adjacent attachment for attached clusters, set correctly below + if (!(mClusterAttachment[mClusters[i].num] & attachProtect)) { // Protected entries are kept on purpose: resetting them could release clusters of the initial tracks that were used for the seeding, and removing such clusters can make the tracking not find the track again + mClusterAttachment[mClusters[i].num] = 0; // Reset adjacent attachment for attached clusters, set correctly below } } } @@ -1948,11 +1945,12 @@ GPUd() void GPUTPCGMMerger::Finalize1(int32_t nBlocks, int32_t nThreads, int32_t } for (uint32_t j = 0; j < trk.NClusters(); j++) { int32_t id = mClusters[trk.FirstClusterRef() + j].num; - uint32_t weight = mTrackOrderAttach[i] | attachAttached; + uint32_t weight = (mTrackOrderAttach[i] << nPureFlags) | attachAttached; uint8_t clusterState = mClusters[trk.FirstClusterRef() + j].state; if (!(clusterState & GPUTPCGMMergedTrackHit::flagReject)) { weight |= attachGood; - } else if (clusterState & GPUTPCGMMergedTrackHit::flagHighIncl) { + } + if (clusterState & GPUTPCGMMergedTrackHit::flagHighIncl) { weight |= attachHighIncl; } if (trk.Leg() == 0) { @@ -1970,7 +1968,7 @@ GPUd() void GPUTPCGMMerger::Finalize2(int32_t nBlocks, int32_t nThreads, int32_t { for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nThreads * nBlocks) { if (mClusterAttachment[i] != 0) { - mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | mTrackSort[mClusterAttachment[i] & attachTrackMask]; + mClusterAttachment[i] = (mClusterAttachment[i] & attachFlagMask) | (mTrackSort[getAttachedTrackId(mClusterAttachment[i])] << nPureFlags); } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index d22fd2e05e0a0..2ce927a5a5f7d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -206,7 +206,7 @@ void GPUTPCGMMerger::DumpFitPrepare(std::ostream& out) const if (j++ % 10 == 0) { out << "\n Cluster attachment "; } - out << i << ": " << (mClusterAttachment[i] & attachTrackMask) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; + out << i << ": " << getAttachedTrackId(mClusterAttachment[i]) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; } } out << "\n"; @@ -255,7 +255,7 @@ void GPUTPCGMMerger::DumpFinal(std::ostream& out) const if (++j % 10 == 0) { out << " Cluster attachment "; } - out << i << ": " << (mClusterAttachment[i] & attachTrackMask) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; + out << i << ": " << getAttachedTrackId(mClusterAttachment[i]) << " / " << (mClusterAttachment[i] & attachFlagMask) << " - "; if (j % 10 == 0) { out << "\n"; } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index ba6abedbcc1ad..7dd7d31a4f9cb 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -27,11 +27,14 @@ enum attachTypes { attachProtect = 0x80000000, attachGood = 0x20000000, attachGoodLeg = 0x10000000, attachTube = 0x08000000, - attachHighIncl = 0x04000000, - attachTrackMask = 0x03FFFFFF, - attachFlagMask = 0xFC000000, + attachHighIncl = 0x1, + attachTrackMask = 0x07FFFFFE, + attachFlagMask = 0xF8000001, + nPureFlags = 1, // Number of flag bits below the track id (do not influence which track wins the attachment) attachZero = 0 }; +GPUdi() static constexpr uint32_t getAttachedTrackId(uint32_t v) { return (v & attachTrackMask) >> nPureFlags; } + struct InterpolationErrorHit { float posY, posZ; GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 2a89e3814f824..a8b13a7bfe480 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -1155,7 +1155,7 @@ GPUd() void GPUTPCGMTrackParam::FindAdjacentClustersUncorrectedPos(const GPUTPCG const int32_t nBinsY = row.Grid().Ny(); const int32_t idOffset = tracker.Data().ClusterIdOffset(); const int32_t* ids = &(tracker.Data().ClusterDataIndex()[row.HitNumberOffset()]); - uint32_t myWeight = merger.TrackOrderAttach()[iTrack] | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; + uint32_t myWeight = (merger.TrackOrderAttach()[iTrack] << gputpcgmmergertypes::nPureFlags) | gputpcgmmergertypes::attachAttached | gputpcgmmergertypes::attachTube; GPUAtomic(uint32_t)* const weights = merger.ClusterAttachment(); if (goodLeg) { myWeight |= gputpcgmmergertypes::attachGoodLeg; diff --git a/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx b/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx index 7e85a14674319..4af2af4f3147c 100644 --- a/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx +++ b/GPU/GPUTracking/display/render/GPUDisplayDraw.cxx @@ -170,7 +170,7 @@ void GPUDisplay::DrawClusters(int32_t iSector) } else if ((mCfgH.markAdjacentClusters & 16) && (attach & gputpcgmmergertypes::attachHighIncl)) { select = tMARKED; } else if (mCfgH.markAdjacentClusters & 8) { - if (fabsf(mIOPtrs->mergedTracks[attach & gputpcgmmergertypes::attachTrackMask].GetParam().GetQPt()) > 20.f) { + if (fabsf(mIOPtrs->mergedTracks[gputpcgmmergertypes::getAttachedTrackId(attach)].GetParam().GetQPt()) > 20.f) { select = tMARKED; } } From 2d620b6e0033f4b90f1f3fb0510616362c9c97ed Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 09:28:25 +0100 Subject: [PATCH 44/54] GPU QA: Add histograms for adjacent clusters without rejection, Fix some naming --- GPU/GPUTracking/qa/GPUQA.cxx | 83 +++++++++++++++++++++++++++--------- GPU/GPUTracking/qa/GPUQA.h | 18 ++++---- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index fd0fb18b6dfa3..13bafef79312d 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -100,11 +100,11 @@ inline checkClusterStateResult GPUQA::checkClusterState(uint32_t attach, T* coun { checkClusterStateResult r; r.unattached = attach == 0; - r.id = attach & gputpcgmmergertypes::attachTrackMask; + r.id = gputpcgmmergertypes::getAttachedTrackId(attach); if (!r.unattached && !(attach & gputpcgmmergertypes::attachProtect)) { r.qpt = fabsf(mTracking->mIOPtrs.mergedTracks[r.id].GetParam().GetQPt()); r.lowPt = r.qpt * mTracking->GetParam().qptB5Scaler > mTracking->GetParam().rec.tpc.rejectQPtB5; - r.mev200 = r.qpt > 5; + r.mev200 = r.qpt * mTracking->GetParam().qptB5Scaler > 5; r.mergedLooperUnconnected = mTracking->mIOPtrs.mergedTracks[r.id].MergedLooperUnconnected(); r.mergedLooperConnected = mTracking->mIOPtrs.mergedTracks[r.id].MergedLooperConnected(); } @@ -172,9 +172,9 @@ static const constexpr char* AXIS_TITLES[5] = {"#it{y}-#it{y}_{mc} (mm) (Resolut static const constexpr char* AXIS_TITLES_NATIVE[5] = {"#it{y}-#it{y}_{mc} (mm) (Resolution)", "#it{z}-#it{z}_{mc} (mm) (Resolution)", "sin(#phi)-sin(#phi_{mc}) (Resolution)", "tan(#lambda)-tan(#lambda_{mc}) (Resolution)", "q*(q/#it{p}_{T} - q/#it{p}_{Tmc}) (Resolution)"}; static const constexpr char* AXIS_TITLES_PULL[5] = {"#it{y}-#it{y}_{mc}/#sigma_{y} (Pull)", "#it{z}-#it{z}_{mc}/#sigma_{z} (Pull)", "sin(#phi)-sin(#phi_{mc})/#sigma_{sin(#phi)} (Pull)", "tan(#lambda)-tan(#lambda_{mc})/#sigma_{tan(#lambda)} (Pull)", "q*(q/#it{p}_{T} - q/#it{p}_{Tmc})/#sigma_{q/#it{p}_{T}} (Pull)"}; -static const constexpr char* CLUSTER_NAMES[GPUQA::N_CLS_HIST] = {"Correctly attached clusters", "Fake attached clusters", "Attached + adjacent clusters", "Fake adjacent clusters", "Clusters of reconstructed tracks", "Used in Physics", "Protected", "All clusters"}; +static const constexpr char* CLUSTER_NAMES[GPUQA::N_CLS_HIST] = {"Correctly attached clusters", "Fake attached clusters", "Attached + adjacent clusters", "Fake adjacent clusters", "Adjacent clusters (not rejected)", "Fake adjacent clusters (not rejected)", "All MC clusters of reconstructed tracks", "Clusters used in Physics", "Protected clusters", "All clusters"}; static const constexpr char* CLUSTER_TITLES[GPUQA::N_CLS_TYPE] = {"Clusters Pt Distribution / Attachment", "Clusters Pt Distribution / Attachment (relative to all clusters)", "Clusters Pt Distribution / Attachment (integrated)"}; -static const constexpr char* CLUSTER_NAMES_SHORT[GPUQA::N_CLS_HIST] = {"Attached", "Fake", "AttachAdjacent", "FakeAdjacent", "FoundTracks", "Physics", "Protected", "All"}; +static const constexpr char* CLUSTER_NAMES_SHORT[GPUQA::N_CLS_HIST] = {"Attached", "Fake", "AttachAdjacent", "FakeAdjacent", "AdjacentNoRej", "FakeAdjacentNoRej", "FoundTracks", "Physics", "Protected", "All"}; static const constexpr char* CLUSTER_TYPES[GPUQA::N_CLS_TYPE] = {"", "Ratio", "Integral"}; static const constexpr char* REJECTED_NAMES[3] = {"All", "Rejected", "Fraction"}; static const constexpr int32_t COLORS_HEX[COLORCOUNT] = {0xB03030, 0x00A000, 0x0000C0, 0x9400D3, 0x19BBBF, 0xF25900, 0x7F7F7F, 0xFFD700, 0x07F707, 0x07F7F7, 0xF08080, 0x000000}; @@ -1021,19 +1021,26 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } if (!mTrackMCLabels[i].isValid()) { for (uint32_t k = 0; k < track->NClusters(); k++) { + int32_t hitId = mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num; if (mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { + if (mTracking->mIOPtrs.mergedTrackHitAttachment && gputpcgmmergertypes::getAttachedTrackId(mTracking->mIOPtrs.mergedTrackHitAttachment[hitId]) == i) { + mClusterParam[hitId].rejected = 1; + } continue; } - mClusterParam[mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num].fakeAttached++; + mClusterParam[hitId].fakeAttached++; } continue; } if (mMCTrackMin == -1 || (label.getTrackID() >= mMCTrackMin && label.getTrackID() < mMCTrackMax)) { for (uint32_t k = 0; k < track->NClusters(); k++) { + int32_t hitId = mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num; if (mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { + if (mTracking->mIOPtrs.mergedTrackHitAttachment && gputpcgmmergertypes::getAttachedTrackId(mTracking->mIOPtrs.mergedTrackHitAttachment[hitId]) == i) { + mClusterParam[hitId].rejected = 1; + } continue; } - int32_t hitId = mTracking->mIOPtrs.mergedTrackHits[track->FirstClusterRef() + k].num; bool correct = false; for (int32_t j = 0; j < GetMCLabelNID(hitId); j++) { if (label == GetMCLabel(hitId, j)) { @@ -1087,7 +1094,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (mClusterParam[i].attached == 0 && mClusterParam[i].fakeAttached == 0) { int32_t attach = mTracking->mIOPtrs.mergedTrackHitAttachment[i]; if (attach & gputpcgmmergertypes::attachFlagMask) { - int32_t track = attach & gputpcgmmergertypes::attachTrackMask; + if ((attach & gputpcgmmergertypes::attachProtect) && !(attach & gputpcgmmergertypes::attachGoodLeg)) { + mClusterParam[i].rejected = 1; + } + int32_t track = gputpcgmmergertypes::getAttachedTrackId(attach); mcLabelI_t trackL = mTrackMCLabels[track]; bool fake = true; for (int32_t j = 0; j < GetMCLabelNID(i); j++) { @@ -1544,7 +1554,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (!track.OK()) { continue; } - if (!mTrackMCLabels[iTrk].isValid()) { + if (!mTrackMCLabels[iTrk].isValid()) { // Track is pure noise / bogus MC label for (uint32_t k = 0; k < track.NClusters(); k++) { if (mTracking->mIOPtrs.mergedTrackHits[track.FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { continue; @@ -1601,7 +1611,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (mMCTrackMin != -1 && (label.getTrackID() < mMCTrackMin || label.getTrackID() >= mMCTrackMax)) { continue; } - for (uint32_t k = 0; k < track.NClusters(); k++) { + for (uint32_t k = 0; k < track.NClusters(); k++) { // Clusters of reconstructed tracks if (mTracking->mIOPtrs.mergedTrackHits[track.FirstClusterRef() + k].state & GPUTPCGMMergedTrackHit::flagReject) { continue; } @@ -1621,8 +1631,20 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (correct) { mClusters[CL_attached]->Fill(pt, weight); mClusters[CL_tracks]->Fill(pt, weight); - } else { + } else { // cluster belongs to other particle mClusters[CL_fake]->Fill(pt, weight); + float totalWeight = 0.f; + for (int32_t j = 0; j < GetMCLabelNID(hitId); j++) { + if (!GetMCLabel(hitId, j).isFake() && GetMCTrackObj(mMCParam, GetMCLabel(hitId, j)).pt > 1.f / mTracking->GetParam().rec.maxTrackQPtB5) { + totalWeight += GetMCLabelWeight(hitId, j); + } + } + for (int32_t j = 0; j < GetMCLabelNID(hitId) && totalWeight > 0.f; j++) { + const mcLabelI_t labelCl = GetMCLabel(hitId, j); + if (!labelCl.isFake() && GetMCTrackObj(mMCParam, labelCl).pt > 1.f / mTracking->GetParam().rec.maxTrackQPtB5 && GetMCTrackObj(mRecTracks, labelCl)) { + mClusters[CL_tracks]->Fill(CAMath::Max(GetMCTrackObj(mMCParam, labelCl).pt, PT_MIN_CLUST), GetMCLabelWeight(hitId, j) * weight / totalWeight); + } + } } mClusters[CL_att_adj]->Fill(pt, weight); mClusters[CL_all]->Fill(pt, weight); @@ -1636,7 +1658,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } } } - for (uint32_t i = 0; i < GetNMCLabels(); i++) { + for (uint32_t i = 0; i < GetNMCLabels(); i++) { // Loop over all clusters if ((mMCTrackMin != -1 && GetMCLabelID(i, 0) < mMCTrackMin) || (mMCTrackMax != -1 && GetMCLabelID(i, 0) >= mMCTrackMax)) { continue; } @@ -1645,9 +1667,9 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } int32_t attach = mTracking->mIOPtrs.mergedTrackHitAttachment[i]; const auto& r = checkClusterState(attach); - if (mClusterParam[i].adjacent) { - int32_t label = mTracking->mIOPtrs.mergedTrackHitAttachment[i] & gputpcgmmergertypes::attachTrackMask; - if (!mTrackMCLabels[label].isValid()) { + if (mClusterParam[i].adjacent) { // Cluster is adjacent + int32_t label = gputpcgmmergertypes::getAttachedTrackId(mTracking->mIOPtrs.mergedTrackHitAttachment[i]); + if (!mTrackMCLabels[label].isValid()) { // Track is pure noise / has bogus MC label float totalWeight = 0.; for (int32_t j = 0; j < GetMCLabelNID(i); j++) { mcLabelI_t labelT = GetMCLabel(i, j); @@ -1669,6 +1691,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } mClusters[CL_att_adj]->Fill(pt, GetMCLabelWeight(i, j) * weight); mClusters[CL_fakeAdj]->Fill(pt, GetMCLabelWeight(i, j) * weight); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(pt, GetMCLabelWeight(i, j) * weight); + mClusters[CL_fakeAdjNoReject]->Fill(pt, GetMCLabelWeight(i, j) * weight); + } mClusters[CL_all]->Fill(pt, GetMCLabelWeight(i, j) * weight); if (r.protect || r.physics) { mClusters[CL_prot]->Fill(pt, GetMCLabelWeight(i, j) * weight); @@ -1681,6 +1707,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } else { mClusters[CL_att_adj]->Fill(0.f, 1.f); mClusters[CL_fakeAdj]->Fill(0.f, 1.f); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(0.f, 1.f); + mClusters[CL_fakeAdjNoReject]->Fill(0.f, 1.f); + } mClusters[CL_all]->Fill(0.f, 1.f); mClusterCounts.nUnaccessible++; if (r.protect || r.physics) { @@ -1697,6 +1727,9 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } mClusters[CL_att_adj]->Fill(pt, 1.f); mClusters[CL_tracks]->Fill(pt, 1.f); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(pt, 1.f); + } mClusters[CL_all]->Fill(pt, 1.f); if (r.protect || r.physics) { mClusters[CL_prot]->Fill(pt, 1.f); @@ -1705,7 +1738,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx mClusters[CL_physics]->Fill(pt, 1.f); } } - } else { + } else { // Cluster is not adjacent float totalWeight = 0.; for (int32_t j = 0; j < GetMCLabelNID(i); j++) { mcLabelI_t labelT = GetMCLabel(i, j); @@ -1724,6 +1757,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx float weight = GetMCLabelWeight(i, j) / totalWeight; if (mClusterParam[i].fakeAdjacent) { mClusters[CL_fakeAdj]->Fill(pt, weight); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(pt, weight); + mClusters[CL_fakeAdjNoReject]->Fill(pt, weight); + } } if (mClusterParam[i].fakeAdjacent) { mClusters[CL_att_adj]->Fill(pt, weight); @@ -1743,6 +1780,10 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx } else { if (mClusterParam[i].fakeAdjacent) { mClusters[CL_fakeAdj]->Fill(0.f, 1.f); + if (!mClusterParam[i].rejected) { + mClusters[CL_adjNoRej]->Fill(0.f, 1.f); + mClusters[CL_fakeAdjNoReject]->Fill(0.f, 1.f); + } } if (mClusterParam[i].fakeAdjacent) { mClusters[CL_att_adj]->Fill(0.f, 1.f); @@ -1791,7 +1832,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx uint32_t jNext = 0; for (uint32_t j = 0; j < track.NClusters(); j = jNext) { uint32_t rowClCount = !(trackClusters[track.FirstClusterRef() + j].state & GPUTPCGMMergedTrackHit::flagReject); - for (jNext = j + 1; j < track.NClusters(); jNext++) { + for (jNext = j + 1; jNext < track.NClusters(); jNext++) { if (trackClusters[track.FirstClusterRef() + j].sector != trackClusters[track.FirstClusterRef() + jNext].sector || trackClusters[track.FirstClusterRef() + j].row != trackClusters[track.FirstClusterRef() + jNext].row) { break; } @@ -1898,7 +1939,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx if (totalWeight > 0 && 10.f * weight400 >= totalWeight) { if (!r.unattached && !r.protect && !r.physics) { mClusterCounts.nFakeRemove400++; - int32_t totalFake = weight400 < 0.9f * totalWeight; + int32_t totalFake = weight400 >= 0.9f * totalWeight; if (totalFake) { mClusterCounts.nFullFakeRemove400++; } @@ -1971,7 +2012,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx uint32_t attach = mTracking->mIOPtrs.mergedTrackHitAttachment[clid]; float x = 0, y = 0, z = 0; if (attach & gputpcgmmergertypes::attachFlagMask) { - uint32_t track = attach & gputpcgmmergertypes::attachTrackMask; + uint32_t track = gputpcgmmergertypes::getAttachedTrackId(attach); const auto& trk = mTracking->mIOPtrs.mergedTracks[track]; mTracking->GetTPCTransform()->Transform(i, j, cl.getPad(), cl.getTime(), x, y, z, trk.GetParam().GetTOffset()); mTracking->GetParam().Sector2Global(i, x, y, z, &x, &y, &z); @@ -1984,7 +2025,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector* tracksEx memset((void*)&trk, 0, sizeof(trk)); memset((void*)&trkHit, 0, sizeof(trkHit)); if (attach & gputpcgmmergertypes::attachFlagMask) { - uint32_t track = attach & gputpcgmmergertypes::attachTrackMask; + uint32_t track = gputpcgmmergertypes::getAttachedTrackId(attach); trk = mTracking->mIOPtrs.mergedTracks[track]; for (uint32_t l = 0; l < trk.NClusters(); l++) { const auto& tmp = mTracking->mIOPtrs.mergedTrackHits[trk.FirstClusterRef() + l]; @@ -3310,8 +3351,8 @@ int32_t GPUQA::DoClusterCounts(uint64_t* attachClusterCounts, int32_t mode) PrintClusterCount(mode, num, "Fake Protect (< 40 MeV)", mClusterCounts.nFakeProtect40, mClusterCounts.nBelow40); } if (mcPresent() && (mQATasks & taskTrackStatistics)) { - PrintClusterCount(mode, num, "Correctly Attached all-trk normalized", mClusterCounts.nCorrectlyAttachedNormalized, mClusterCounts.nTotal); - PrintClusterCount(mode, num, "Correctly Attached non-fake normalized", mClusterCounts.nCorrectlyAttachedNormalizedNonFake, mClusterCounts.nTotal); + PrintClusterCount(mode, num, "Correctly Attached all-trk row-normalized", mClusterCounts.nCorrectlyAttachedNormalized, mClusterCounts.nTotal); + PrintClusterCount(mode, num, "Correctly Attached non-fake row-normalized", mClusterCounts.nCorrectlyAttachedNormalizedNonFake, mClusterCounts.nTotal); } if (mTextDump) { fclose(mTextDump); diff --git a/GPU/GPUTracking/qa/GPUQA.h b/GPU/GPUTracking/qa/GPUQA.h index f257c9a418659..f236498a858c6 100644 --- a/GPU/GPUTracking/qa/GPUQA.h +++ b/GPU/GPUTracking/qa/GPUQA.h @@ -143,7 +143,7 @@ class GPUQA int32_t loadHistograms(std::vector& i1, std::vector& i2, std::vector& i3, std::vector& i4, int32_t tasks = -1); void* AllocateScratchBuffer(size_t nBytes); - static constexpr int32_t N_CLS_HIST = 8; + static constexpr int32_t N_CLS_HIST = 10; static constexpr int32_t N_CLS_TYPE = 3; static constexpr int32_t MC_LABEL_INVALID = -1e9; @@ -156,7 +156,7 @@ class GPUQA }; struct additionalClusterParameters { - int32_t attached, fakeAttached, adjacent, fakeAdjacent; + int32_t attached, fakeAttached, adjacent, fakeAdjacent, rejected; float pt; }; @@ -254,7 +254,7 @@ class GPUQA std::vector mClusterParam; int32_t mNTotalFakes = 0; - TH1F* mEff[6][2][2][5]; // eff,clone,fake,all,all-fake - findable - secondaries - y,z,phi,eta,pt - work,result + TH1F* mEff[6][2][2][5]; // eff,clone,fake,all,all-fake,all-filled-internal - findable - secondaries - y,z,phi,eta,pt - work,result TGraphAsymmErrors* mEffResult[4][2][2][5]; TCanvas* mCEff[6]; TPad* mPEff[6][4]; @@ -276,11 +276,13 @@ class GPUQA CL_fake = 1, CL_att_adj = 2, CL_fakeAdj = 3, - CL_tracks = 4, - CL_physics = 5, - CL_prot = 6, - CL_all = 7 }; - TH1D* mClusters[N_CLS_TYPE * N_CLS_HIST - 1]; // attached, fakeAttached, attach+adjacent, fakeAdjacent, physics, protected, tracks, all / count, rel, integral + CL_adjNoRej = 4, + CL_fakeAdjNoReject = 5, + CL_tracks = 6, + CL_physics = 7, + CL_prot = 8, + CL_all = 9 }; + TH1D* mClusters[N_CLS_TYPE * N_CLS_HIST - 1]; // attached, fakeAttached, attach+adjacent, fakeAdjacent, adjacent not rejected, fakeAdjacent not rejected, tracks, physics, protected, all / count, rel, integral TCanvas* mCClust[N_CLS_TYPE]; TPad* mPClust[N_CLS_TYPE]; TLegend* mLClust[N_CLS_TYPE]; From 8fe99e53d61c173d9d17e912a43d0a59a82aad74 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 18:20:17 +0100 Subject: [PATCH 45/54] GPU TPC: Add tpcSortClusters option --- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 + GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx | 4 ++-- GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index 10095eb69ec2c..aa0469604fcc4 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -432,6 +432,7 @@ AddOption(tpcUseOldCPUDecoding, bool, false, "", 0, "Enable old CPU-based TPC de AddOption(tpcApplyCFCutsAtDecoding, bool, false, "", 0, "Apply cluster cuts from clusterization during decoding of compressed clusters") AddOption(tpcApplyClusterFilterOnCPU, uint8_t, 0, "", 0, "Apply custom cluster filter of GPUTPCClusterFilter class, 0: off, 1: debug, 2: PbPb23") AddOption(tpcWriteClustersAfterRejection, bool, false, "", 0, "Apply TPC rejection strategy before writing clusters") +AddOption(tpcSortClusters, bool, false, "", 0, "Enforce sorting TPC clusters") AddOption(oclPlatformNum, int32_t, -1, "", 0, "Platform to use, in case the backend provides multiple platforms (OpenCL only, -1 = auto-select, -2 query all platforms (also incompatible))") // TODO: Create some backend-specific options AddOption(oclCompileFromSources, bool, false, "", 0, "Compile OpenCL binary from included source code instead of using included spirv code") AddOption(oclOverrideSourceBuildFlags, std::string, "", "", 0, "Override OCL build flags for compilation from source, put a space for empty options") diff --git a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx index 8ce90c740804a..485cde456f227 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx @@ -941,9 +941,9 @@ int32_t GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) std::unique_ptr tmpNativeClusterBuffer; const bool buildNativeGPU = doGPU && NeedTPCClustersOnGPU(); - const bool buildNativeHost = (mRec->GetRecoStepsOutputs() & gpudatatypes::InOutType::TPCClusters) || GetProcessingSettings().deterministicGPUReconstruction; // TODO: Should do this also when clusters are needed for later steps on the host but not requested as output + const bool buildNativeHost = (mRec->GetRecoStepsOutputs() & gpudatatypes::InOutType::TPCClusters) || GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().tpcSortClusters; // TODO: Should do this also when clusters are needed for later steps on the host but not requested as output const bool propagateMCLabels = buildNativeHost && GetProcessingSettings().runMC && processors()->ioPtrs.tpcPackedDigits && processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC; - const bool sortClusters = buildNativeHost && (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4); + const bool sortClusters = buildNativeHost && (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters); if (GetProcessingSettings().runMC && (!processors()->ioPtrs.tpcPackedDigits || !processors()->ioPtrs.tpcPackedDigits->tpcDigitsMC)) { GPUWarning("Requested to process MC labels, but no labels present"); diff --git a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx index 6ca50f24351c4..fafefa7803188 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx @@ -431,7 +431,7 @@ int32_t GPUChainTracking::RunTPCDecompression() GPUMemCpy(RecoStep::TPCDecompression, mInputsHost->mPclusterNativeOutput, DecompressorShadow.mNativeClustersBuffer, sizeof(Decompressor.mNativeClustersBuffer[0]) * nClustersFinal, unattachedStream, false); SynchronizeStream(unattachedStream); } - if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4) { + if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters) { runKernel(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression)); const ClusterNativeAccess* decoded = mIOPtrs.clustersNative; if (doGPU) { From 35768af3e46aafbc524e1bd5248452272b1e483b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 19:02:39 +0100 Subject: [PATCH 46/54] GPU Standalone / QA: Add testSyncAsyncMC option to pass cluster MC labels to async pass --- .../DataCompression/GPUTPCCompression.h | 1 + .../GPUTPCCompressionKernels.cxx | 8 ++- GPU/GPUTracking/DataTypes/GPUDataTypesIO.h | 1 + .../Definitions/GPUSettingsListStandalone.h | 1 + GPU/GPUTracking/Global/GPUChainTracking.h | 1 + .../Global/GPUChainTrackingCompression.cxx | 68 ++++++++++++++----- .../Standalone/Benchmark/standalone.cxx | 21 ++++-- 7 files changed, 76 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompression.h b/GPU/GPUTracking/DataCompression/GPUTPCCompression.h index 852156d7ab7f0..b628af3c7dde1 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompression.h +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompression.h @@ -65,6 +65,7 @@ class GPUTPCCompression : public GPUProcessor void DumpCompressedClusters(std::ostream& out); #endif GPUd() bool rejectCluster(int32_t idx, const GPUParam& param, const GPUTrackingInOutPointers& ioPtrs) const; + GPUd() static bool isClusterRejected(int32_t idx, const GPUParam& param, const GPUTrackingInOutPointers& ioPtrs); protected: struct memory { diff --git a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx index 67dacf993f0a4..588340bd73e64 100644 --- a/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx +++ b/GPU/GPUTracking/DataCompression/GPUTPCCompressionKernels.cxx @@ -189,9 +189,11 @@ GPUd() bool GPUTPCCompressionKernels::GPUTPCCompressionKernels_Compare<4>::opera GPUd() bool GPUTPCCompression::rejectCluster(int32_t idx, const GPUParam& GPUrestrict() param, const GPUTrackingInOutPointers& GPUrestrict() ioPtrs) const { - if (mClusterStatus[idx]) { - return true; - } + return mClusterStatus[idx] || isClusterRejected(idx, param, ioPtrs); +} + +GPUd() bool GPUTPCCompression::isClusterRejected(int32_t idx, const GPUParam& GPUrestrict() param, const GPUTrackingInOutPointers& GPUrestrict() ioPtrs) +{ int32_t attach = ioPtrs.mergedTrackHitAttachment[idx]; bool unattached = attach == 0; diff --git a/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h b/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h index b541df40a23e5..dd93996078163 100644 --- a/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h +++ b/GPU/GPUTracking/DataTypes/GPUDataTypesIO.h @@ -195,6 +195,7 @@ struct GPUTrackingInOutPointers { uint32_t nOutputClusRefsTPCO2 = 0; const o2::MCCompLabel* outputTracksTPCO2MC = nullptr; const o2::tpc::CompressedClustersFlat* tpcCompressedClusters = nullptr; + const o2::dataformats::ConstMCTruthContainerView* tpcCompressedClustersMC = nullptr; const o2::tpc::ClusterNativeAccess* clustersNativeReduced = nullptr; // TPC links diff --git a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h index 6ff3f8ec38487..be67272379b10 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h @@ -82,6 +82,7 @@ AddOption(printSettings, int32_t, 0, "", 0, "Print all settings", def(1)) AddOption(testSyncAsync, bool, false, "syncAsync", 0, "Test first synchronous and then asynchronous processing") AddOption(testSync, bool, false, "sync", 0, "Test settings for synchronous phase") AddOption(testSyncAsyncQcInSync, bool, false, "syncAsyncSyncQC", 0, "Run QC in sync phase of testSyncAsync") +AddOption(testSyncAsyncMC, bool, false, "", 0, "Propagate the TPC cluster MC labels of the clusters surviving the compression to the asynchronous phase of testSyncAsync (enforces tpcSortClusters)") AddOption(timeFrameTime, bool, false, "tfTime", 0, "Print some debug information about time frame processing time") AddOption(controlProfiler, bool, false, "", 0, "Issues GPU profiler stop and start commands to profile only the relevant processing part") AddOption(preloadEvents, bool, false, "", 0, "Preload events into host memory before start processing") diff --git a/GPU/GPUTracking/Global/GPUChainTracking.h b/GPU/GPUTracking/Global/GPUChainTracking.h index 92eda50194c74..1fef646f803c4 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.h +++ b/GPU/GPUTracking/Global/GPUChainTracking.h @@ -159,6 +159,7 @@ class GPUChainTracking : public GPUChain const GPUQA* GetQA() const { return mQAFromForeignChain ? mQAFromForeignChain->mQA.get() : mQA.get(); } GPUQA* GetQA() { return mQAFromForeignChain ? mQAFromForeignChain->mQA.get() : mQA.get(); } int32_t ForceInitQA(); + void GenerateCompressionMCLabels(const GPUChainTracking& chainSync); void SetQAFromForeignChain(GPUChainTracking* chain) { mQAFromForeignChain = chain; } const GPUSettingsDisplay* GetEventDisplayConfig() const { return mConfigDisplay; } const GPUSettingsQA* GetQAConfig() const { return mConfigQA; } diff --git a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx index fafefa7803188..5d7a785b7a25e 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingCompression.cxx @@ -253,7 +253,7 @@ int32_t GPUChainTracking::RunTPCDecompression() auto& decompressTimer = getTimer("TPCDecompression", 0); auto allocatorUse = runFiltering ? std::function{allocatorTmp} : std::function{allocatorFinal}; decompressTimer.Start(); - if (decomp.decompress(mIOPtrs.tpcCompressedClusters, *mClusterNativeAccess, allocatorUse, param(), GetProcessingSettings().deterministicGPUReconstruction)) { + if (decomp.decompress(mIOPtrs.tpcCompressedClusters, *mClusterNativeAccess, allocatorUse, param(), GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters)) { GPUError("Error decompressing clusters"); return 1; } @@ -433,15 +433,8 @@ int32_t GPUChainTracking::RunTPCDecompression() } if (GetProcessingSettings().deterministicGPUReconstruction || GetProcessingSettings().debugLevel >= 4 || GetProcessingSettings().tpcSortClusters) { runKernel(GetGridAutoStep(unattachedStream, RecoStep::TPCDecompression)); - const ClusterNativeAccess* decoded = mIOPtrs.clustersNative; if (doGPU) { - for (uint32_t i = 0; i < NSECTORS; i++) { - for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { - ClusterNative* begin = mInputsHost->mPclusterNativeOutput + decoded->clusterOffset[i][j]; - ClusterNative* end = begin + decoded->nClusters[i][j]; - std::sort(begin, end); - } - } + SortClusters(false, false, mClusterNativeAccess.get(), mInputsHost->mPclusterNativeOutput); } SynchronizeStream(unattachedStream); } @@ -453,10 +446,58 @@ int32_t GPUChainTracking::RunTPCDecompression() tmpParam.reset(); WriteConstantParams(); } + if (mIOPtrs.tpcCompressedClustersMC) { + if (mIOPtrs.tpcCompressedClustersMC->getIndexedSize() != mIOPtrs.clustersNative->nClustersTotal) { + GPUError("Number of MC labels for compressed clusters (%u) does not match number of decoded clusters (%u)", (uint32_t)mIOPtrs.tpcCompressedClustersMC->getIndexedSize(), (uint32_t)mIOPtrs.clustersNative->nClustersTotal); + return 1; + } + mClusterNativeAccess->clustersMCTruth = mIOPtrs.tpcCompressedClustersMC; + } DoDebugDump(GPUChainTrackingDebugFlags::TPCDecompressedClusters, &GPUChainTracking::DumpClusters, *mDebugFile, mIOPtrs.clustersNative); return 0; } +namespace +{ +template +void writeReducedClusterLabels(const ClusterNativeAccess& clusters, std::pair labelBuffer, T&& rejectCluster) +{ + o2::dataformats::MCLabelContainer tmpContainer; + uint32_t nOutput = 0; + for (uint32_t i = 0; i < clusters.nClustersTotal; i++) { + if (!rejectCluster(i)) { + for (const auto& element : clusters.clustersMCTruth->getLabels(i)) { + tmpContainer.addElement(nOutput, element); + } + nOutput++; + } + } + tmpContainer.flatten_to(*labelBuffer.first); + *labelBuffer.second = *labelBuffer.first; +} +} // namespace + +void GPUChainTracking::GenerateCompressionMCLabels(const GPUChainTracking& chainSync) +{ + const GPUTrackingInOutPointers& ioPtrs = chainSync.mIOPtrs; + if (!ioPtrs.clustersNative || !ioPtrs.clustersNative->clustersMCTruth || !ioPtrs.mergedTrackHitAttachment || !ioPtrs.mergedTracks) { + throw std::runtime_error("Cannot generate MC labels for compressed clusters, need clusters with MC labels and merged tracks with cluster attachment of the sync chain"); + } + for (uint32_t i = 0; i < NSECTORS; i++) { // The decoded clusters get sorted, so the labels match only if the clusters before the compression were sorted as well + for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { + for (uint32_t k = 1; k < ioPtrs.clustersNative->nClusters[i][j]; k++) { + if (ioPtrs.clustersNative->clusters[i][j][k] < ioPtrs.clustersNative->clusters[i][j][k - 1]) { + throw std::runtime_error("Cannot generate MC labels for compressed clusters, clusters of sync chain are not sorted"); + } + } + } + } + mIOMem.clusterNativeMCView = std::make_unique(); + mIOMem.clusterNativeMCBuffer = std::make_unique(); + writeReducedClusterLabels(*ioPtrs.clustersNative, {mIOMem.clusterNativeMCBuffer.get(), mIOMem.clusterNativeMCView.get()}, [&chainSync, &ioPtrs](uint32_t i) { return GPUTPCCompression::isClusterRejected(i, chainSync.GetParam(), ioPtrs); }); + mIOPtrs.tpcCompressedClustersMC = mIOMem.clusterNativeMCView.get(); +} + void GPUChainTracking::WriteReducedClusters() { GPUTPCCompression& Compressor = processors()->tpcCompressor; @@ -491,21 +532,14 @@ void GPUChainTracking::WriteReducedClusters() } nOutput = 0; - o2::dataformats::MCLabelContainer tmpContainer; for (uint32_t i = 0; i < mIOPtrs.clustersNative->nClustersTotal; i++) { if (!Compressor.rejectCluster(i, param(), mIOPtrs)) { - if (mIOPtrs.clustersNative->clustersMCTruth) { - for (const auto& element : mIOPtrs.clustersNative->clustersMCTruth->getLabels(i)) { - tmpContainer.addElement(nOutput, element); - } - } clBuffer[nOutput++] = mIOPtrs.clustersNative->clustersLinear[i]; } } mIOPtrs.clustersNativeReduced = mClusterNativeAccessReduced.get(); if (mIOPtrs.clustersNative->clustersMCTruth) { - tmpContainer.flatten_to(*labelBuffer.first); - *labelBuffer.second = *labelBuffer.first; + writeReducedClusterLabels(*mIOPtrs.clustersNative, labelBuffer, [this, &Compressor](uint32_t i) { return Compressor.rejectCluster(i, param(), mIOPtrs); }); mClusterNativeAccessReduced->clustersMCTruth = labelBuffer.second; } } diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 652ec5ccc23f2..925b2c584857e 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -208,6 +208,13 @@ int32_t ReadConfiguration(int argc, char** argv) printf("Cannot run asynchronous processing with double pipeline\n"); return 1; } + if (configStandalone.testSyncAsyncMC) { + if (!configStandalone.testSyncAsync || configStandalone.QA.noMC) { + printf("testSyncAsyncMC requires syncAsync and cannot be used with QA.noMC\n"); + return 1; + } + configStandalone.proc.tpcSortClusters = true; + } if (configStandalone.proc.doublePipeline && (configStandalone.runs < 4 || !configStandalone.outputcontrolmem)) { printf("Double pipeline mode needs at least 4 runs per event and external output. To cycle though multiple events, use --preloadEvents and --runs n for n iterations round-robin\n"); return 1; @@ -531,7 +538,7 @@ int32_t SetupReconstruction() steps.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); steps.inputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, true); steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, false); - procSet.runMC = false; + procSet.runMC = configStandalone.testSyncAsyncMC; procSet.runQA = runAsyncQA; procSet.eventDisplay = eventDisplay.get(); procSet.runCompressionStatistics = 0; @@ -598,10 +605,11 @@ int32_t ReadEvent(int32_t n) } #if !defined(GPUCA_RUN2) && defined(GPUCA_BUILD_QA) && defined(GPUCA_STANDALONE) if ((configStandalone.proc.runQA || configStandalone.eventDisplay) && !configStandalone.QA.noMC) { - chainTracking->ForceInitQA(); - chainTracking->GetQA()->UpdateChain(chainTracking); - if (chainTracking->GetQA()->ReadO2MCData((eventsDir + "mc." + std::to_string(n) + ".dump").c_str()) && - chainTracking->GetQA()->ReadO2MCData((eventsDir + "mc.0.dump").c_str()) && configStandalone.proc.runQA) { + GPUChainTracking* chain = (configStandalone.testSyncAsyncMC && !configStandalone.testSyncAsyncQcInSync) ? chainTrackingAsync : chainTracking; + chain->ForceInitQA(); + chain->GetQA()->UpdateChain(chain); + if (chain->GetQA()->ReadO2MCData((eventsDir + "mc." + std::to_string(n) + ".dump").c_str()) && + chain->GetQA()->ReadO2MCData((eventsDir + "mc.0.dump").c_str()) && configStandalone.proc.runQA) { throw std::runtime_error("Error reading O2 MC dump"); } } @@ -753,6 +761,9 @@ int32_t RunBenchmark(GPUReconstruction* recUse, GPUChainTracking* chainTrackingU chainTrackingAsync->mIOPtrs.nRawClusters[i] = 0; } chainTrackingAsync->mIOPtrs.clustersNative = nullptr; + if (configStandalone.testSyncAsyncMC) { + chainTrackingAsync->GenerateCompressionMCLabels(*chainTracking); + } if (configStandalone.runsInit > 0 && configStandalone.proc.debugCSV.empty()) { recAsync->SetResetTimers(iRun < configStandalone.runsInit); } From 941fa0f4ca342802b2c05cc29a3eb6d345302d15 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 22:28:25 +0100 Subject: [PATCH 47/54] GPU Standalone: add testSyncAsyncNoSyncSettings option --- .../Definitions/GPUSettingsListStandalone.h | 1 + .../Standalone/Benchmark/standalone.cxx | 80 ++++++++++--------- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h index be67272379b10..20018a2792cc0 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsListStandalone.h @@ -82,6 +82,7 @@ AddOption(printSettings, int32_t, 0, "", 0, "Print all settings", def(1)) AddOption(testSyncAsync, bool, false, "syncAsync", 0, "Test first synchronous and then asynchronous processing") AddOption(testSync, bool, false, "sync", 0, "Test settings for synchronous phase") AddOption(testSyncAsyncQcInSync, bool, false, "syncAsyncSyncQC", 0, "Run QC in sync phase of testSyncAsync") +AddOption(testSyncAsyncNoSyncSettings, bool, false, "", 0, "Do not apply the settings for synchronous processing in testSyncAsync / testSync") AddOption(testSyncAsyncMC, bool, false, "", 0, "Propagate the TPC cluster MC labels of the clusters surviving the compression to the asynchronous phase of testSyncAsync (enforces tpcSortClusters)") AddOption(timeFrameTime, bool, false, "tfTime", 0, "Print some debug information about time frame processing time") AddOption(controlProfiler, bool, false, "", 0, "Issues GPU profiler stop and start commands to profile only the relevant processing part") diff --git a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx index 925b2c584857e..62d53f7bb9580 100644 --- a/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx +++ b/GPU/GPUTracking/Standalone/Benchmark/standalone.cxx @@ -453,6 +453,7 @@ int32_t SetupReconstruction() } steps.steps = gpudatatypes::RecoStep::AllRecoSteps; + steps.outputs.clear(); if (configStandalone.runTRD != -1) { steps.steps.setBits(gpudatatypes::RecoStep::TRDTracking, configStandalone.runTRD > 0); } else if (chainTracking->GetTRDGeometry() == nullptr) { @@ -485,7 +486,6 @@ int32_t SetupReconstruction() steps.steps.setBits(gpudatatypes::RecoStep::TPCCompression, false); steps.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); steps.steps.setBits(gpudatatypes::RecoStep::TPCDecompression, true); - steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, false); } else if (grp.needsClusterer) { steps.inputs.setBits(gpudatatypes::InOutType::TPCRaw, true); steps.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); @@ -493,8 +493,12 @@ int32_t SetupReconstruction() steps.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); } + GPUSettingsRec recSetAsync = recSet; + GPUSettingsProcessing procSetAsync = procSet; + GPURecoStepConfiguration stepsAsync = steps; + // Set settings for synchronous - GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, configStandalone.testSyncAsync || configStandalone.testSync, configStandalone.rundEdx); + GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, (configStandalone.testSyncAsync || configStandalone.testSync) && !configStandalone.testSyncAsyncNoSyncSettings, configStandalone.rundEdx); int32_t runAsyncQA = procSet.runQA && !configStandalone.testSyncAsyncQcInSync ? procSet.runQA : 0; if (configStandalone.testSyncAsync) { procSet.eventDisplay = nullptr; @@ -503,26 +507,26 @@ int32_t SetupReconstruction() } } - // Apply --recoSteps flag last so it takes precedence - // E.g. ApplySyncSettings might enable TPCdEdx, but might not be needed if only clusterizer was requested - if (configStandalone.recoSteps >= 0) { - steps.steps &= configStandalone.recoSteps; - } - if (configStandalone.recoStepsGPU >= 0) { - steps.stepsGPUMask &= configStandalone.recoStepsGPU; - } + const auto finalizeSteps = [](GPURecoStepConfiguration& stepsCfg, GPUSettingsProcessing& procCfg) { + if (configStandalone.recoSteps >= 0) { + stepsCfg.steps &= configStandalone.recoSteps; + } + if (configStandalone.recoStepsGPU >= 0) { + stepsCfg.stepsGPUMask &= configStandalone.recoStepsGPU; + } - steps.outputs.clear(); - steps.outputs.setBits(gpudatatypes::InOutType::TPCMergedTracks, steps.steps.isSet(gpudatatypes::RecoStep::TPCMerging)); - steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, steps.steps.isSet(gpudatatypes::RecoStep::TPCCompression)); - steps.outputs.setBits(gpudatatypes::InOutType::TRDTracks, steps.steps.isSet(gpudatatypes::RecoStep::TRDTracking)); - steps.outputs.setBits(gpudatatypes::InOutType::TPCClusters, steps.steps.isSet(gpudatatypes::RecoStep::TPCClusterFinding)); - - if (steps.steps.isSet(gpudatatypes::RecoStep::TRDTracking)) { - if (procSet.createO2Output && !procSet.trdTrackModelO2) { - procSet.createO2Output = 1; // Must not be 2, to make sure TPC GPU tracks are still available for TRD + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TPCMergedTracks, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCMerging)); + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCCompression)); + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TRDTracks, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TRDTracking)); + stepsCfg.outputs.setBits(gpudatatypes::InOutType::TPCClusters, stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCClusterFinding) || stepsCfg.steps.isSet(gpudatatypes::RecoStep::TPCDecompression)); + + if (stepsCfg.steps.isSet(gpudatatypes::RecoStep::TRDTracking)) { + if (procCfg.createO2Output && !procCfg.trdTrackModelO2) { + procCfg.createO2Output = 1; // Must not be 2, to make sure TPC GPU tracks are still available for TRD + } } - } + }; + finalizeSteps(steps, procSet); rec->SetSettings(&grp, &recSet, &procSet, &steps); if (configStandalone.proc.doublePipeline) { @@ -530,25 +534,25 @@ int32_t SetupReconstruction() } if (configStandalone.testSyncAsync) { // TODO: Add --async mode / flag // Set settings for asynchronous - steps.steps.setBits(gpudatatypes::RecoStep::TPCDecompression, true); - steps.steps.setBits(gpudatatypes::RecoStep::TPCdEdx, true); - steps.steps.setBits(gpudatatypes::RecoStep::TPCCompression, false); - steps.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); - steps.inputs.setBits(gpudatatypes::InOutType::TPCRaw, false); - steps.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); - steps.inputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, true); - steps.outputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, false); - procSet.runMC = configStandalone.testSyncAsyncMC; - procSet.runQA = runAsyncQA; - procSet.eventDisplay = eventDisplay.get(); - procSet.runCompressionStatistics = 0; - if (recSet.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { - procSet.tpcInputWithClusterRejection = 1; + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCDecompression, true); + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCdEdx, true); + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCCompression, false); + stepsAsync.steps.setBits(gpudatatypes::RecoStep::TPCClusterFinding, false); + stepsAsync.inputs.setBits(gpudatatypes::InOutType::TPCRaw, false); + stepsAsync.inputs.setBits(gpudatatypes::InOutType::TPCClusters, false); + stepsAsync.inputs.setBits(gpudatatypes::InOutType::TPCCompressedClusters, true); + procSetAsync.runMC = configStandalone.testSyncAsyncMC; + procSetAsync.runQA = runAsyncQA; + procSetAsync.eventDisplay = eventDisplay.get(); + procSetAsync.runCompressionStatistics = 0; + if (recSetAsync.tpc.rejectionStrategy >= GPUSettings::RejectionStrategyB) { + procSetAsync.tpcInputWithClusterRejection = 1; } - recSet.tpc.disableMarkAdjacent = 0xFF; - recSet.maxTrackQPtB5 = CAMath::Min(recSet.maxTrackQPtB5, recSet.tpc.rejectQPtB5); - GPUChainTracking::ApplySyncSettings(procSet, recSet, steps.steps, false, configStandalone.rundEdx); - recAsync->SetSettings(&grp, &recSet, &procSet, &steps); + recSetAsync.tpc.disableMarkAdjacent = 0xFF; + recSetAsync.maxTrackQPtB5 = CAMath::Min(recSetAsync.maxTrackQPtB5, recSetAsync.tpc.rejectQPtB5); + GPUChainTracking::ApplySyncSettings(procSetAsync, recSetAsync, stepsAsync.steps, false, configStandalone.rundEdx); + finalizeSteps(stepsAsync, procSetAsync); + recAsync->SetSettings(&grp, &recSetAsync, &procSetAsync, &stepsAsync); } if (configStandalone.outputcontrolmem) { From 7438ba45949d186cc84e19e82811d6f6d46ccd01 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 17 Nov 2026 23:06:41 +0100 Subject: [PATCH 48/54] GPU QA: Add extra histograms for pt > 50 / 200 MeV --- GPU/GPUTracking/qa/GPUQA.cxx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/GPU/GPUTracking/qa/GPUQA.cxx b/GPU/GPUTracking/qa/GPUQA.cxx index 13bafef79312d..bd0deed3f0fd3 100644 --- a/GPU/GPUTracking/qa/GPUQA.cxx +++ b/GPU/GPUTracking/qa/GPUQA.cxx @@ -2825,7 +2825,7 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } } - uint64_t attachClusterCounts[N_CLS_HIST]; + uint64_t attachClusterCounts[N_CLS_HIST + 4]; // + correctly attached / all clusters in the pt bins entirely above 50 MeV and above 200 MeV if (mQATasks & taskClusterAttach) { // Process Cluster Attachment Histograms if (mConfig.inputHistogramsOnly == 0) { @@ -2849,6 +2849,17 @@ int32_t GPUQA::DrawQAHistograms(TObjArray* qcout) } attachClusterCounts[i] = val; } + for (int32_t i = 0; i < 2; i++) { + double valAttached = 0, valAll = 0; + for (int32_t j = 1; j < mClusters[CL_all]->GetXaxis()->GetNbins() + 2; j++) { + if (mClusters[CL_all]->GetXaxis()->GetBinLowEdge(j) >= (i ? 0.2 : 0.05)) { + valAttached += mClusters[CL_attached]->GetBinContent(j); + valAll += mClusters[CL_all]->GetBinContent(j); + } + } + attachClusterCounts[N_CLS_HIST + 2 * i] = valAttached; + attachClusterCounts[N_CLS_HIST + 2 * i + 1] = valAll; + } if (!CLUST_HIST_INT_SUM) { for (int32_t i = 0; i < N_CLS_HIST; i++) { @@ -3324,6 +3335,8 @@ int32_t GPUQA::DoClusterCounts(uint64_t* attachClusterCounts, int32_t mode) PrintClusterCount(mode, num, "Unattached", attachClusterCounts[N_CLS_HIST - 1] - attachClusterCounts[CL_att_adj], mClusterCounts.nTotal); PrintClusterCount(mode, num, "Removed (Strategy A)", attachClusterCounts[CL_att_adj] - attachClusterCounts[CL_prot], mClusterCounts.nTotal); // Attached + Adjacent (also fake) - protected PrintClusterCount(mode, num, "Unaccessible", mClusterCounts.nUnaccessible, mClusterCounts.nTotal); // No contribution from track >= 10 MeV, unattached or fake-attached/adjacent + PrintClusterCount(mode, num, "Correctly attached (> 50 MeV)", attachClusterCounts[N_CLS_HIST], attachClusterCounts[N_CLS_HIST + 1]); // Normalized to all clusters of particles above the threshold + PrintClusterCount(mode, num, "Correctly attached (> 200 MeV)", attachClusterCounts[N_CLS_HIST + 2], attachClusterCounts[N_CLS_HIST + 3]); } else { PrintClusterCount(mode, num, "All Clusters", mClusterCounts.nTotal, mClusterCounts.nTotal); PrintClusterCount(mode, num, "Used in Physics", mClusterCounts.nPhysics, mClusterCounts.nTotal); From 18a8b41110627e737bfeb557e5a7b82c122d344a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 14:45:25 +0100 Subject: [PATCH 49/54] GPU Math: Add support for half-precision min/max in CUDA --- GPU/Common/GPUCommonMath.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/GPU/Common/GPUCommonMath.h b/GPU/Common/GPUCommonMath.h index 62efdfb80bb35..2e542d027307f 100644 --- a/GPU/Common/GPUCommonMath.h +++ b/GPU/Common/GPUCommonMath.h @@ -236,6 +236,19 @@ class GPUCommonMath typedef GPUCommonMath CAMath; +#ifdef __CUDACC__ +template <> +GPUhd() half GPUCommonMath::Min(const half x, const half y) // BUG: CUDA __hmin / __hmax is not constexpr, fix once fixed in CUDA +{ + return __hmin(x, y); +} +template <> +GPUhd() half GPUCommonMath::Max(const half x, const half y) +{ + return __hmax(x, y); +} +#endif + template GPUhdi() constexpr float GPUCommonMath::Sum2(float w, Args... args) { From ec7b77bb93e7739faa2177275ae05fa79714f98b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 23:33:33 +0100 Subject: [PATCH 50/54] GPU CPUDevice: Add consistency check for some RTC parameters --- .../Base/cpudevice/GPUReconstructionCPUDevice.cxx | 12 +++++++++--- .../cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx index 846dd32674be9..149bb75c52792 100644 --- a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx +++ b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDevice.cxx @@ -39,9 +39,7 @@ GPUReconstructionCPUDevice::GPUReconstructionCPUDevice(const GPUSettingsDeviceBa { if (mMaster == nullptr) { mInternals = new GPUReconstructionCPUDeviceInternals; - *mParDevice = *mParCPU; // We run the CPU kernels on the "device", so we use the CPU kernel parameters - mParDevice->par_ALTERNATE_BORDER_SORT = 1; // Required by the GPU code path without RTC - mParDevice->par_CF_SCAN_WORKGROUP_SIZE = 1; // No block-wide scan with 1 thread per block, the clusterizer uses the CPU stream compaction + *mParDevice = *mParCPU; // We run the host-compiled CPU kernels on the "device", so the device parameters must be the CPU kernel parameters. Other values are possible only with RTC (genRTC), checked in InitDevice_Runtime for (auto& v : mParDevice->par_LB_maxThreads) { v = 1; } @@ -72,6 +70,14 @@ int32_t GPUReconstructionCPUDevice::InitDevice_Runtime() exit(0); } + if (mProcessingSettings->alternateBorderSort == -1) { + mProcessingSettings->alternateBorderSort = 1; // Required by the GPU code path without RTC + } + if (!GetProcessingSettings().rtc.enable && memcmp(&mParDevice->par_LB_forceBlocks + 1, &mParCPU->par_LB_forceBlocks + 1, (const char*)(mParCPU + 1) - (const char*)(&mParCPU->par_LB_forceBlocks + 1))) { // All members behind the launch bounds + GPUError("CPUDevice kernel parameters differ from the parameters the host kernels were compiled with, this is possible only with RTC"); + return 1; + } + if (mMaster == nullptr) { mMultiprocessorCount = std::max(1, mMaxHostThreads); // One device block runs as one TBB task mThreadCount = 1; diff --git a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx index 90a765b85cba3..afa6b24098972 100644 --- a/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx +++ b/GPU/GPUTracking/Base/cpudevice/GPUReconstructionCPUDeviceGenRTC.cxx @@ -60,6 +60,7 @@ int32_t GPUReconstructionCPUDevice::genRTC(std::string& filename, uint32_t& nCom const std::string baseCommand = rtc::baseCommand(proc, std::string_view(_binary_GPUReconstructionCPUDevicertc_command_start, _binary_GPUReconstructionCPUDevicertc_command_len)) + " " + proc.rtctech.overrideArchitecture; const std::string noFastMath(_binary_GPUReconstructionCPUDevicertc_command_no_fast_math_start, _binary_GPUReconstructionCPUDevicertc_command_no_fast_math_len); + mParDevice->par_CF_SCAN_WORKGROUP_SIZE = 1; // Must match the 1 thread per block for the scan kernels to compile, the clusterizer uses the CPU stream compaction rtc::loadLaunchBounds(proc, *mParDevice); const std::string launchBounds = rtc::launchBoundsCode(proc, *mParDevice, 0, mWarpSize); if (proc.rtctech.printLaunchBounds >= 2) { From 5b5d9789556f6e747865192d945cac2ee7f419dc Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Nov 2026 11:51:24 +0100 Subject: [PATCH 51/54] GPU TPC: synchronize threads in warp between fit iterations to reduce warp divergence --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 1 + 1 file changed, 1 insertion(+) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index a8b13a7bfe480..7fd2414a6365e 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -94,6 +94,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const int8_t adjacentLeg = 1 + (track.Leg() == 0); for (int32_t iWay = rebuilt ? (wayRebuildStart + 2) : 0; iWay < nWays; iWay++) { // TODO DR: Unrolling has no performance improvement on GPU, why? + GPUbarrierWarp(); // TODO: Validate on different GPUs, check if we should do similar things in other places int32_t nMissed = 0, nMissedHard = 0; // nMissed: cluster rejected, fit failure. nMissedHard: also propagation failure / high incl. float sumInvSqrtCharge = 0.f; // TODO: Compute in first iteration and store! int32_t nAvgCharge = 0; From 68affbc96182a142cdb97aa39632de9fd099f4c2 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 19:17:43 +0100 Subject: [PATCH 52/54] GPU: Provide separate fit kernels for rebuild/retry, remove mergerSortTracks runtime option --- GPU/GPUTracking/Base/GPUReconstruction.cxx | 3 --- GPU/GPUTracking/Definitions/GPUGetConstexpr.h | 2 +- GPU/GPUTracking/Definitions/GPUSettingsList.h | 1 - .../Definitions/Parameters/GPUParameters.csv | 4 +++- .../Parameters/GPUParameters.json.example | 16 ++++++++++++- .../Global/GPUChainTrackingMerger.cxx | 8 +++---- GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 3 +-- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx | 24 +++++++++++++++---- GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h | 5 +++- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 16 ++++++++++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 6 +++-- GPU/GPUTracking/kernels.cmake | 4 +++- 12 files changed, 67 insertions(+), 25 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx index c1a09f4075bc9..e0385664a5a00 100644 --- a/GPU/GPUTracking/Base/GPUReconstruction.cxx +++ b/GPU/GPUTracking/Base/GPUReconstruction.cxx @@ -301,9 +301,6 @@ int32_t GPUReconstruction::InitPhaseBeforeDevice() if (!GetProcessingSettings().createO2Output || !IsGPU()) { mProcessingSettings->clearO2OutputFromGPU = false; } - if (!(mRecoSteps.stepsGPUMask & gpudatatypes::RecoStep::TPCMerging)) { - mProcessingSettings->mergerSortTracks = false; - } if (GetProcessingSettings().debugLevel > 3 || !IsGPU() || GetProcessingSettings().deterministicGPUReconstruction) { mProcessingSettings->delayedOutput = false; } diff --git a/GPU/GPUTracking/Definitions/GPUGetConstexpr.h b/GPU/GPUTracking/Definitions/GPUGetConstexpr.h index b70890738e1b7..5d6d9613e54bd 100644 --- a/GPU/GPUTracking/Definitions/GPUGetConstexpr.h +++ b/GPU/GPUTracking/Definitions/GPUGetConstexpr.h @@ -22,7 +22,7 @@ // This is a temporary workaround required for clang (with c++20), until we can go to C++23 with P2280R4, which allows getting constexpr static values from references -#if defined(__clang__) && __cplusplus >= 202002L && __cplusplus < 202302L +#if (defined(__clang__) && __cplusplus >= 202002L && __cplusplus < 202302L) || defined(__CPUDEVICE__) // BUG: The workaround is needed for GCC if used inside a templated function. namespace o2::gpu::internal { diff --git a/GPU/GPUTracking/Definitions/GPUSettingsList.h b/GPU/GPUTracking/Definitions/GPUSettingsList.h index aa0469604fcc4..f73791ea99cd5 100644 --- a/GPU/GPUTracking/Definitions/GPUSettingsList.h +++ b/GPU/GPUTracking/Definitions/GPUSettingsList.h @@ -397,7 +397,6 @@ AddOption(nStreams, int8_t, 8, "", 0, "Number of GPU streams / command queues") AddOption(nTPCClustererLanes, int8_t, -1, "", 0, "Number of TPC clusterers that can run in parallel (-1 = autoset)") AddOption(overrideClusterizerFragmentLen, int32_t, -1, "", 0, "Force the cluster max fragment len to a certain value (-1 = autodetect)") AddOption(delayedOutput, bool, true, "", 0, "Delay output to be parallel to track fit") -AddOption(mergerSortTracks, int8_t, -1, "", 0, "Sort track indizes for GPU track fit (-1 = auto-device based on device parameters)") AddOption(alternateBorderSort, int8_t, -1, "", 0, "Alternative implementation for sorting of border tracks (-1 = auto-device based on device parameters)") AddOption(tpcCompressionGatherMode, int8_t, -1, "", 0, "TPC Compressed Clusters Gather Mode (0: DMA transfer gather gpu to host, 1: serial DMA to host and gather by copy on CPU, 2. gather via GPU kernal DMA access, 3. gather on GPU via kernel, dma afterwards, -1 = auto-device based on device parameters)") AddOption(tpcCompressionGatherModeKernel, int8_t, -1, "", 0, "TPC Compressed Clusters Gather Mode Kernel (0: unbufferd, 1-3: buffered, 4: multi-block, -1 = auto-device based on device parameters)") diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv index 5892f4acfe9c4..b197c20130738 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.csv @@ -27,7 +27,9 @@ GPUTPCCFDecodeZSLink,"""GPUCA_WARP_SIZE""",,"""GPUCA_WARP_SIZE""","""GPUCA_WARP_ GPUTPCCFDecodeZSDenseLink,"""GPUCA_WARP_SIZE""",,"[""GPUCA_WARP_SIZE"", 4]","[""GPUCA_WARP_SIZE"", 14]",,,,,,"[""GPUCA_WARP_SIZE"", 14]","""GPUCA_WARP_SIZE""","[""GPUCA_WARP_SIZE"", 22]","[""GPUCA_WARP_SIZE"", 22]",,"[""GPUCA_WARP_SIZE"", 17]",,"[""GPUCA_WARP_SIZE"", 8]","[""GPUCA_WARP_SIZE"", 5]", GPUTPCCFGather,"[1024, 1]",,"[1024, 5]","[1024, 1]",,,,,,"[160, 11]","[1024, 1]",736,896,,"[928, 1]",,"[1024, 1]","[320, 2]", COMPRESSION_GATHER,1024,,1024,1024,,,,,,1024,1024,,1024,,,,,, -GPUTPCGMMergerTrackFit,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" +GPUTPCGMMergerTrackFit_def,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" +GPUTPCGMMergerTrackFit_rebuild,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" +GPUTPCGMMergerTrackFit_retry,256,,"[192, 2]","[64, 7]",,,,,,"[32, 16]","[32, 8]","[32, 14]","[160, 2]",,"[32, 24]",,"[64, 8]","[64, 6]","[64, 4]" GPUTPCGMMergerFollowLoopers,256,,"[256, 5]","[256, 4]",,,,,,"[256, 4]","[128, 4]","[1024, 1]",640,,"[128, 16]",,"[224, 3]","[256, 7]","[32, 64]" GPUTPCGMMergerSectorRefit,256,,"[64, 4]","[256, 2]",,,,,,"[32, 8]","[64, 5]","[32, 7]","[32, 7]",,"[32, 20]",,"[32, 10]","[64, 4]","[256, 8]" GPUTPCGMMergerUnpackResetIds,256,,256,256,,,,,,256,256,256,256,,256,,256,256, diff --git a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example index 15f905cbcc41e..7390c4b520ae9 100644 --- a/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example +++ b/GPU/GPUTracking/Definitions/Parameters/GPUParameters.json.example @@ -167,7 +167,21 @@ "AMPERE": 1024, "TURING": 1024 }, - "GPUTPCGMMergerTrackFit": { + "GPUTPCGMMergerTrackFit_def": { + "default": 256, + "MI100": [192, 2], + "VEGA": [64, 7], + "AMPERE": [64, 4], + "TURING": [32, 8] + }, + "GPUTPCGMMergerTrackFit_rebuild": { + "default": 256, + "MI100": [192, 2], + "VEGA": [64, 7], + "AMPERE": [64, 4], + "TURING": [32, 8] + }, + "GPUTPCGMMergerTrackFit_retry": { "default": 256, "MI100": [192, 2], "VEGA": [64, 7], diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 6786f7fc927e9..3fbf2453fe700 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -196,7 +196,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) waitForTransfer = 1; } - const bool mergerSortTracks = GetProcessingSettings().mergerSortTracks == -1 ? mRec->getGPUParameters(doGPU).par_SORT_BEFORE_FIT : GetProcessingSettings().mergerSortTracks; + const bool mergerSortTracks = mRec->getGPUParameters(doGPU).par_SORT_BEFORE_FIT; if (mergerSortTracks) { runKernel(GetGridAuto(0, deviceType)); CondWaitEvent(waitForTransfer, &mEvents->single); @@ -236,7 +236,7 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); } // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 0); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); if (param().rec.tpc.rebuildTrackInFit) { runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.HitWeights(), Merger.NClusters() * sizeof(*MergerShadow.HitWeights())); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); @@ -250,10 +250,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpInterpolatedHits, *mDebugFile); runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), 0); DoDebugAndDump(RecoStep::TPCMerging, GPUChainTrackingDebugFlags::TPCMergingRefit, Merger, &GPUTPCGMMerger::DumpRebuiltTracks, *mDebugFile); - runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0), mergerSortTracks ? 1 : 0, 1); + runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); } if (param().rec.tpc.retryRefit) { - runKernel(GetGridAuto(0), -1, param().rec.tpc.rebuildTrackInFit); + runKernel(GetGridAuto(0)); } runKernel(GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 31a6b9a18a10b..614f73679180f 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -430,8 +430,7 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem) void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) { computePointerWithAlignment(mem, mTrackOrderAttach, mNMaxTracks); - const bool mergerSortTracks = mRec->GetProcessingSettings().mergerSortTracks == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_SORT_BEFORE_FIT : mRec->GetProcessingSettings().mergerSortTracks; - if (mergerSortTracks) { + if (mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCMerging).par_SORT_BEFORE_FIT) { computePointerWithAlignment(mem, mTrackOrderProcess, mNMaxTracks); } return mem; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx index 2ef0d0ba59748..48780cb060d90 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.cxx @@ -18,16 +18,30 @@ using namespace o2::gpu; -template <> -GPUdii() void GPUTPCGMMergerTrackFit::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger, int32_t mode, int32_t rebuilt) +template +GPUdii() void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { - const int32_t iEnd = mode == -1 ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); + constexpr bool rebuilt = I == GPUTPCGMMergerTrackFit::rebuild, retry = I == GPUTPCGMMergerTrackFit::retry; + const int32_t iEnd = retry ? merger.Memory()->nRetryRefit : merger.NMergedTracks(); GPUCA_TBB_KERNEL_LOOP(merger.GetRec(), int32_t, ii, iEnd, { - const int32_t i = mode == -1 ? merger.RetryRefitIds()[ii] : (mode ? merger.TrackOrderProcess()[ii] : ii); - GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger, rebuilt, mode == -1); + int32_t i; + if constexpr (retry) { + i = merger.RetryRefitIds()[ii]; + } else if constexpr (GPUCA_PAR_SORT_BEFORE_FIT > 0) { + i = merger.TrackOrderProcess()[ii]; + } else { + i = ii; + } + (GPUTPCGMTrackParam::RefitTrack(merger.MergedTracks()[i], i, merger)); // Parentheses, since the comma of the template arguments would split the macro arguments }); } +#ifndef GPUCA_GPUCODE // The host needs explicit instantiations, the kernels are called from another compilation unit +template void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); +template void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); +template void GPUTPCGMMergerTrackFit::Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); +#endif + template <> GPUdii() void GPUTPCGMMergerFollowLoopers::Thread<0>(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& GPUrestrict() merger) { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h index 82605f47a5f27..023a72f276de3 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerGPU.h @@ -35,8 +35,11 @@ class GPUTPCGMMergerGeneral : public GPUKernelTemplate class GPUTPCGMMergerTrackFit : public GPUTPCGMMergerGeneral { public: + enum K { def = 0, + rebuild = 1, + retry = 2 }; template - GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger, int32_t mode, int32_t rebuilt); + GPUd() static void Thread(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& smem, processorType& merger); }; class GPUTPCGMMergerFollowLoopers : public GPUTPCGMMergerGeneral diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 7fd2414a6365e..46254350d1393 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -56,11 +56,13 @@ static thread_local bool CADEBUG_CONDITION = true; using namespace o2::gpu; using namespace o2::tpc; -GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track, bool rebuilt, bool retryAttempt) +template +GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_t iTrk, int32_t& GPUrestrict() N, int32_t& GPUrestrict() NTolerated, float& GPUrestrict() Alpha, GPUTPCGMMergedTrack& GPUrestrict() track) { constexpr float maxSinPhi = constants::MAX_SIN_PHI; const GPUParam& GPUrestrict() param = merger.Param(); + const bool rebuilt = retryAttempt ? param.rec.tpc.rebuildTrackInFit : rebuiltKernel; // The retry runs on the rebuilt tracks if the rebuild is enabled const float maxSinForUpdate = param.rec.tpc.maxSinPhiForFitUpdate; GPUTPCGMMergedTrackHit* GPUrestrict() clusters = merger.Clusters() + track.FirstClusterRef(); @@ -1488,11 +1490,13 @@ GPUd() bool GPUTPCGMTrackParam::CheckNumericalQuality(float overrideCovYY) const return ok; } -GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger, bool rebuilt, bool retryAttempt) +template +GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() track, int32_t iTrk, GPUTPCGMMerger& GPUrestrict() merger) { if (!track.OK()) { return; } + const bool rebuilt = retryAttempt ? merger.Param().rec.tpc.rebuildTrackInFit : rebuiltKernel; CADEBUG3(if (GPUCA_CADEBUG_ENABLED >= 2) { static const std::set dbgTracks = []() { std::set m; if (const char* e = getenv("DEBUG_TRACK")) { std::string s(e); for (char* t = strtok(s.data(), ";, \n\t"); t; t = strtok(nullptr, ";, \n\t")) { m.insert(atoi(t)); } } return m; }(); CADEBUG_CONDITION = dbgTracks.count(iTrk); }); @@ -1500,7 +1504,7 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() int32_t NTolerated = 0; // Clusters not fit but tollerated for track length cut GPUTPCGMTrackParam t = track.Param(); float Alpha = track.Alpha(); - bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track, rebuilt, retryAttempt); + bool ok = t.Fit(merger, iTrk, nTrackHits, NTolerated, Alpha, track); CADEBUG(if (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) printf("Finished Fit Track %7d --- OUTPUT hits %d -> %d+%d = %d, QPt %f -> %f, SP %f, OK %d chi2 %f chi2ndf %f\n", iTrk, track.NClusters(), nTrackHits, NTolerated, nTrackHits + NTolerated, track.GetParam().GetQPt(), t.QPt(), t.SinPhi(), (int32_t)ok, t.Chi2(), t.Chi2() / CAMath::Max(1, nTrackHits))); if (!ok && (!merger.Param().rec.tpc.rebuildTrackInFit || rebuilt) && !retryAttempt && merger.Param().rec.tpc.retryRefit) { @@ -1536,6 +1540,12 @@ GPUdii() void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& GPUrestrict() // if (track.OK()) merger.DebugRefitMergedTrack(track); } +#ifndef GPUCA_GPUCODE // The host needs explicit instantiations, RefitTrack is called from another compilation unit +template void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); +template void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); +template void GPUTPCGMTrackParam::RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); +#endif + GPUd() void GPUTPCGMTrackParam::Rotate(float alpha) { float cA, sA; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 2e8ee86ca8196..665c2da4ef14d 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -154,12 +154,14 @@ class GPUTPCGMTrackParam GPUd() bool CheckNumericalQuality(float overrideCovYY = -1.f) const; GPUd() bool CheckCov() const; - GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track, bool rebuilt, bool retryAttempt); + template + GPUd() bool Fit(GPUTPCGMMerger& merger, int32_t iTrk, int32_t& N, int32_t& NTolerated, float& Alpha, GPUTPCGMMergedTrack& track); GPUd() void DodEdx(GPUdEdx& dEdx, GPUdEdx& dEdxAlt, GPUTPCGMMerger& merger, bool finalFit, int ihit, int ihitMergeFirst, int wayDirection, const GPUTPCGMMergedTrackHit* clusters, uint8_t clusterState, float zz, uint8_t dEdxSubThresholdRow GPUCA_DEBUG_STREAMER_CHECK(, int32_t iTrk)); GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); - GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger, bool rebuilt, bool retryAttempt); + template + GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha); GPUd() void MirrorTo(GPUTPCGMPropagator& prop, float toY, float toZ, bool inFlyDirection, const GPUParam& param, uint8_t row, uint8_t clusterState, bool mirrorParameters, int8_t sector); GPUd() int32_t MergeDoubleRowClusters(int32_t& ihit, int32_t wayDirection, GPUTPCGMMergedTrackHit* clusters, const GPUTPCGMMerger& merger, GPUTPCGMPropagator& prop, float& xx, float& yy, float& zz, int32_t maxN, float clAlpha, uint8_t& clusterState, const bool markReject); diff --git a/GPU/GPUTracking/kernels.cmake b/GPU/GPUTracking/kernels.cmake index 539840baaff7a..ade04a4bdfeb7 100644 --- a/GPU/GPUTracking/kernels.cmake +++ b/GPU/GPUTracking/kernels.cmake @@ -51,7 +51,9 @@ o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, mergedTracks2" "= TPC o2_gpu_add_kernel("GPUTPCGlobalDebugSortKernels, borderTracks" "= TPCMERGER" NO int8_t parameter) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fill" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map) o2_gpu_add_kernel("GPUTPCCreateOccupancyMap, fold" "= TPCOCCUPANCY" LB GPUTPCClusterOccupancyMapBin* map uint32_t* output) -o2_gpu_add_kernel("GPUTPCGMMergerTrackFit" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB int32_t mode int32_t rebuilt) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit, def" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit, rebuild" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB) +o2_gpu_add_kernel("GPUTPCGMMergerTrackFit, retry" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT TPCDEDX" LB) o2_gpu_add_kernel("GPUTPCGMMergerFollowLoopers" "GPUTPCGMMergerGPU TPCMERGER TPCTRACKER MATLUT" LB) o2_gpu_add_kernel("GPUTPCGMMergerUnpackResetIds" "GPUTPCGMMergerGPU TPCMERGER" LB int32_t iSector) o2_gpu_add_kernel("GPUTPCGMMergerSectorRefit" "GPUTPCGMMergerGPU TPCMERGER MATLUT" LB int32_t iSector) From d966c19a8f8f32751859573b861640015b338f84 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Wed, 18 Nov 2026 08:15:13 +0100 Subject: [PATCH 53/54] GPU TPC: Refactor rebuild data structures to remove GPU memory usage --- GPU/GPUTracking/Global/GPUChainTracking.cxx | 4 +- .../Global/GPUChainTrackingMerger.cxx | 4 + GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx | 92 ++++++++++++------- GPU/GPUTracking/Merger/GPUTPCGMMerger.h | 29 +++--- GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx | 13 +-- .../Merger/GPUTPCGMMergerRebuildInternal.h | 64 +++++++++++++ GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 30 +++--- 7 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h diff --git a/GPU/GPUTracking/Global/GPUChainTracking.cxx b/GPU/GPUTracking/Global/GPUChainTracking.cxx index 2761fcb3ec733..b66cc7d49dcc9 100644 --- a/GPU/GPUTracking/Global/GPUChainTracking.cxx +++ b/GPU/GPUTracking/Global/GPUChainTracking.cxx @@ -258,8 +258,8 @@ bool GPUChainTracking::ValidateSteps() bool GPUChainTracking::ValidateSettings() { int32_t gatherMode = mRec->GetProcessingSettings().tpcCompressionGatherMode == -1 ? mRec->getGPUParameters(mRec->GetRecoStepsGPU() & gpudatatypes::RecoStep::TPCCompression).par_COMP_GATHER_MODE : mRec->GetProcessingSettings().tpcCompressionGatherMode; - if (param().rec.tpc.rebuildTrackInFit && param().rec.tpc.rebuildTrackInFitClusterCandidates == 0) { - GPUError("Need at least 1 cluster candidate to rebuild tracks during fit"); + if (param().rec.tpc.rebuildTrackInFit && (param().rec.tpc.rebuildTrackInFitClusterCandidates == 0 || param().rec.tpc.rebuildTrackInFitClusterCandidates > 13)) { + GPUError("Need between 1 and 13 cluster candidates to rebuild tracks during fit"); return false; } if (param().continuousMaxTimeBin > (int32_t)GPUSettings::TPC_MAX_TF_TIME_BIN) { diff --git a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx index 3fbf2453fe700..ea72f6aa8e0ce 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx @@ -19,6 +19,7 @@ #include "GPUO2DataTypes.h" #include "GPUQA.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUConstantMem.h" #include "GPUTPCGMMergerGPU.h" #include "GPUTPCGMO2Output.h" @@ -233,7 +234,10 @@ int32_t GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput) } if (param().rec.tpc.rebuildTrackInFit) { + AllocateRegisteredMemory(Merger.MemoryResRebuild()); // Sized by the number of merged tracks, which was transferred to the host above + WriteToConstantMemory(RecoStep::TPCMerging, (char*)&processors()->tpcMerger - (char*)processors(), &MergerShadow, sizeof(MergerShadow), 0); runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.ClusterCandidates(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * param().rec.tpc.rebuildTrackInFitClusterCandidates * sizeof(*MergerShadow.ClusterCandidates())); + runKernel({{numBlocks, -ThreadCount(), 0, deviceType, RecoStep::TPCMerging}}, MergerShadow.TrackRebuildHelperRow(), Merger.NMergedTracks() * GPUTPCGeometry::NROWS * sizeof(*MergerShadow.TrackRebuildHelperRow())); } // mRec->SetNActiveThreads(1); // For GPUTPCGMTrackParam:: GPUCA_CADEBUG: run the host kernels from here on with 1 thread, reset below runKernel(doGPU ? GetGrid(Merger.NMergedTracks(), 0) : GetGridAuto(0)); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx index 614f73679180f..43400445501d2 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.cxx @@ -34,6 +34,7 @@ #include "GPUTPCTracker.h" #include "GPUTPCTrackParam.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUO2DataTypes.h" #include "TPCFastTransformPOD.h" #include "GPUTPCConvertImpl.h" @@ -407,8 +408,6 @@ void* GPUTPCGMMerger::SetPointersMerger(void* mem) computePointerWithAlignment(mem, mLoopData, mNMaxTracks); // GPUTPCGMMergerTrackFit - GPUTPCGMMergerFollowLoopers, Reducing mNMaxTracks does not save memory, other parts are larger anyway if (mRec->GetParam().rec.tpc.rebuildTrackInFit) { computePointerWithAlignment(mem, mSharedCount2, mNClusters); - computePointerWithAlignment(mem, mClusterCandidates, mNMaxTracks * GPUTPCGeometry::NROWS * Param().rec.tpc.rebuildTrackInFitClusterCandidates); - computePointerWithAlignment(mem, mTrackRebuildHelper, mNMaxTracks); computePointerWithAlignment(mem, mHitWeights, mNClusters); } if (mRec->GetParam().rec.tpc.retryRefit) { @@ -427,6 +426,15 @@ void* GPUTPCGMMerger::SetPointersMemory(void* mem) return mem; } +void* GPUTPCGMMerger::SetPointersRebuild(void* mem) +{ + const size_t nTracks = HostProcessor(this).NMergedTracks(); // Allocated once the number of merged tracks is known on the host + computePointerWithAlignment(mem, mClusterCandidates, nTracks * GPUTPCGeometry::NROWS * mRec->GetParam().rec.tpc.rebuildTrackInFitClusterCandidates); + computePointerWithAlignment(mem, mTrackRebuildHelperRow, nTracks * GPUTPCGeometry::NROWS); + computePointerWithAlignment(mem, mTrackRebuildHelper, nTracks); + return mem; +} + void* GPUTPCGMMerger::SetPointersRefitScratch(void* mem) { computePointerWithAlignment(mem, mTrackOrderAttach, mNMaxTracks); @@ -503,6 +511,7 @@ void GPUTPCGMMerger::RegisterMemoryAllocation() mMemoryResOutputO2MC = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersOutputO2MC, GPUMemoryResource::MEMORY_OUTPUT_FLAG | GPUMemoryResource::MEMORY_HOST | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerOutputO2MC"); } } + mMemoryResRebuild = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersRebuild, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK | GPUMemoryResource::MEMORY_CUSTOM, "TPCMergerRebuild"); mMemoryResMemory = mRec->RegisterMemoryAllocation(this, &GPUTPCGMMerger::SetPointersMemory, GPUMemoryResource::MEMORY_PERMANENT, "TPCMergerMemory"); } @@ -1697,7 +1706,7 @@ GPUd() void GPUTPCGMMerger::CollectMergedTracks(int32_t nBlocks, int32_t nThread const GPUTPCTracker& trk = GetConstantMem()->tpcTrackers[t->Sector()]; const GPUTPCHitId& ic = trk.TrackHits()[t->OrigTrack()->FirstHitID() + i]; uint32_t id = trk.Data().ClusterDataIndex(trk.Data().Row(ic.RowIndex()), ic.HitIndex()) + GetConstantMem()->ioPtrs.clustersNative->clusterOffset[t->Sector()][0]; - *c2 = trackCluster{.id = id, .row = (uint8_t)ic.RowIndex(), .sector = t->Sector(), .error = 0.f, .weight = 0, .best = 0}; + *c2 = trackCluster{id, (uint8_t)ic.RowIndex(), t->Sector()}; } nHits += nTrackHits; } @@ -2166,6 +2175,16 @@ GPUd() uint32_t GPUTPCGMMerger::MinHitsForQPt(float qPt, float err2QPt) const return Param().tpcMinHitsB5Abs((CAMath::Abs(qPt) + CAMath::Min(CAMath::Sqrt(CAMath::Max(0.f, err2QPt)), Param().rec.tpc.trackFitMinHitsMaxQPtIncrease * CAMath::Abs(qPt))) * Param().qptB5Scaler); } +GPUd() uint32_t GPUTPCGMMerger::CandidateId(uint32_t clusterId, uint8_t sector, uint8_t row) const +{ + return clusterId - GetConstantMem()->ioPtrs.clustersNative->clusterOffset[sector][row]; +} + +GPUd() uint32_t GPUTPCGMMerger::CandidateClusterId(trackRebuildHelperCandidate candidate, uint8_t row) const +{ + return GetConstantMem()->ioPtrs.clustersNative->clusterOffset[candidate.getSector()][row] + candidate.getId(); +} + GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread) { for (uint32_t i = iBlock * nThreads + iThread; i < mNClusters; i += nBlocks * nThreads) { @@ -2186,10 +2205,10 @@ GPUd() void GPUTPCGMMerger::PrepareHitWeights(int32_t nBlocks, int32_t nThreads, for (uint32_t j = 0; j < trk.NClusters(); j++) { const auto& cl = mClusters[trk.FirstClusterRef() + j]; auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + cl.row) * nMaxCandidates]; - if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].id == 0 || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error + if ((!(cl.state & GPUTPCGMMergedTrackHit::flagReject) || trk.GetParam().GetNDF() <= 0) && (candidates[0].isEmpty() || cl.row == lastRow)) { // Row not searched: the original clusters of the row compete as normal candidates with fixed error for (uint32_t k = 0; k < nMaxCandidates; k++) { - if (candidates[k].id == 0) { - candidates[k] = {.id = cl.num + 2, .row = cl.row, .sector = cl.sector, .error = Param().rec.tpc.rebuildTrackFallbackErr, .weight = 0, .best = 0}; + if (candidates[k].isEmpty()) { + candidates[k].set(CandidateId(cl.num, cl.sector, cl.row), cl.sector, (CAMath::half_t)Param().rec.tpc.rebuildTrackFallbackErr); break; } } @@ -2208,16 +2227,17 @@ GPUd() void GPUTPCGMMerger::ComputeHitWeights(int32_t nBlocks, int32_t nThreads, continue; } for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { - auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best == 0) { - if (candidates[iteration].id >= 2 && (iteration == 0 || candidates[iteration].error <= CAMath::Max(Param().rec.tpc.rebuildTrackAltCandidateGoodErr, candidates[0].error * Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster - float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2) / (1.f + Param().rec.tpc.rebuildErrorWeightFactor * candidates[iteration].error); // Tracks with NDF <= 0 use their nominal number of clusters and assume a fixed chi2 per cluster - int32_t weight = weightf > 0.f ? (weightf > 2e9f ? 2e9f : (int32_t)weightf) : 0; - candidates[iteration].weight = weight; - CAMath::AtomicMax(&mHitWeights[candidates[iteration].id - 2], weight); - CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, candidates[iteration].id - 2, weight)); + auto* c = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; + auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; + if (helper.getBest() == 0) { + if (c[iteration].hasCluster() && (iteration == 0 || c[iteration].error <= CAMath::Max((CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateGoodErr, c[0].error * (CAMath::half_t)Param().rec.tpc.rebuildTrackAltCandidateMaxErrFactor))) { // Don't fall back to a much worse candidate, the best one is then rather a shared cluster + float weightf = (trk.GetParam().GetNDF() > 0 ? (trk.GetParam().GetNDF() + 5) * trk.GetParam().GetNDF() * 64 / trk.GetParam().GetChi2() : (2 * (int32_t)trk.NClusters() - 5) * 128 / Param().rec.tpc.rebuildTrackFallbackTrackChi2); // Nominal number of clusters and fixed chi2 per cluster for NDF <= 0 + weightf /= (1.f + Param().rec.tpc.rebuildErrorWeightFactor * (float)c[iteration].error); + const int32_t weight = helper.setWeight(weightf); + CAMath::AtomicMax(&mHitWeights[CandidateClusterId(c[iteration], j)], weight); + CADEBUG(printf("REBUILD: iTrk %d Iteration %d Row %d Cluster %d Weight %d\n", i, iteration, j, CandidateClusterId(c[iteration], j), weight)); } else { - candidates[0].best = -2; + helper.setBest(trackRebuildHelperRow::bestNoCandidate); } } } @@ -2233,10 +2253,9 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights1(int32_t nBlocks, int32_t nThreads } for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best == 0) { - if (candidates[iteration].weight == mHitWeights[candidates[iteration].id - 2]) { - candidates[0].best = iteration + 1; - } + auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; + if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[iteration], j)]) { + helper.setBest(iteration + 1); } } } @@ -2256,15 +2275,17 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads uint8_t nSharedRowCandidates = 0; for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best == 0 && candidates[nIter - 1].weight == mHitWeights[candidates[nIter - 1].id - 2]) { - candidates[0].best = nIter; + auto& helper = mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j]; // the helper weight is from the last computed iteration: the one that set best, or the last one if best is still 0 + if (helper.getBest() == 0 && helper.getWeight() == mHitWeights[CandidateClusterId(candidates[nIter - 1], j)]) { + helper.setBest(nIter); } - if (candidates[0].best > 0) { + if (helper.getBest() > 0 && helper.getBest() < trackRebuildHelperRow::bestRowRemoved) { nHits++; - if (candidates[candidates[0].best - 1].weight != mHitWeights[candidates[candidates[0].best - 1].id - 2]) { - mSharedCount2[candidates[candidates[0].best - 1].id - 2] = 1; // shared between tracks of different iterations + const uint32_t id = CandidateClusterId(candidates[helper.getBest() - 1], j); + if (helper.getWeight() != mHitWeights[id]) { + mSharedCount2[id] = 1; // shared between tracks of different iterations } - } else if (candidates[0].best != -1 && candidates[0].id >= 2) { + } else if (helper.getBest() != trackRebuildHelperRow::bestRowRemoved && candidates[0].hasCluster()) { sharedRowCandidates[nSharedRowCandidates++] = j; } } @@ -2274,16 +2295,16 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads if (nShared) { const uint32_t n = Param().rec.tpc.rebuildTrackInFitClusterCandidates; const auto* candidates = &mClusterCandidates[i * GPUTPCGeometry::NROWS * n]; - GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates](const uint8_t& a, const uint8_t& b) { + GPUCommonAlgorithm::sort(sharedRowCandidates, sharedRowCandidates + nSharedRowCandidates, [n, candidates, this](const uint8_t& a, const uint8_t& b) { const auto& ca = candidates[a * n]; const auto& cb = candidates[b * n]; - return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? ca.id < cb.id : ca.error < cb.error, ca.error < cb.error); + return GPUCA_DETERMINISTIC_CODE(ca.error == cb.error ? CandidateClusterId(ca, a) < CandidateClusterId(cb, b) : ca.error < cb.error, ca.error < cb.error); }); for (uint32_t j = 0; j < nShared; j++) { auto* cj = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], cj[0].id - 2, cj[0].error)); - cj[0].best = 1; - mSharedCount2[cj[0].id - 2] = 1; // TODO: Should we also recover iter0 clusters? + CADEBUG(printf("REBUILD: iTrk %d Recovering shared %d / %d: row %d %d, error %f\n", i, j, nShared, (int)sharedRowCandidates[j], CandidateClusterId(cj[0], sharedRowCandidates[j]), cj[0].error)); + mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + sharedRowCandidates[j]].setBest(1); + mSharedCount2[CandidateClusterId(cj[0], sharedRowCandidates[j])] = 1; // TODO: Should we also recover iter0 clusters? } nHits += nShared; } @@ -2304,17 +2325,18 @@ GPUd() void GPUTPCGMMerger::ResolveHitWeights2(int32_t nBlocks, int32_t nThreads const bool reverse = mTrackRebuildHelper[i].reverse; // TODO: CE tracks may have clusters on both sides in the rows around the crossing: make sure the track crosses the CE only once, by cutting at the best row and removing the clusters on the wrong side of the cut for (unsigned int j = 0; j < GPUTPCGeometry::NROWS; j++) { const auto& candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].best > 0) { + if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() > 0 && mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() < trackRebuildHelperRow::bestRowRemoved) { auto& outCl = mClusters[iOut + (reverse ? (written) : (nHits - 1 - written))]; - const auto& best = candidates[candidates[0].best - 1]; - const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[best.id - 2]; + const auto& best = candidates[mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() - 1]; + const uint32_t id = CandidateClusterId(best, j); + const ClusterNative& GPUrestrict() cl = GetConstantMem()->ioPtrs.clustersNative->clustersLinear[id]; uint8_t flags = (uint8_t)(cl.getFlags() & GPUTPCGMMergedTrackHit::clustererAndSharedFlags); if ((mTrackRebuildHelper[i].highInclRowLow != 255 && j <= mTrackRebuildHelper[i].highInclRowLow) || (mTrackRebuildHelper[i].highInclRowHigh != 255 && j >= mTrackRebuildHelper[i].highInclRowHigh)) { flags |= GPUTPCGMMergedTrackHit::flagHighIncl; } - outCl = {.num = best.id - 2, .sector = best.sector, .row = (uint8_t)j, .state = flags}; + outCl = {.num = id, .sector = best.getSector(), .row = (uint8_t)j, .state = flags}; written++; - CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, best.id - 2)); + CADEBUG(printf("REBUILD: iTrk %d Assigned Cluster Row %d Hit %d\n", i, j, id)); } } } diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h index 1ca312165f17a..fe3a027799146 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMerger.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMerger.h @@ -25,6 +25,7 @@ #include "GPUTPCGMMergerTypes.h" #include "GPUGeneralKernels.h" #include "GPUTPCGeometry.h" +#include "GPUCommonMath.h" #if !defined(GPUCA_GPUCODE) #include @@ -79,20 +80,15 @@ class GPUTPCGMMerger : public GPUProcessor GPUAtomic(uint32_t) nLooperMatchCandidates; }; - struct trackCluster { // TODO: Reduce size of this struct! + struct trackCluster { uint32_t id; uint8_t row; uint8_t sector; - float error; - int32_t weight; - int32_t best; }; - struct trackRebuildHelper { - uint8_t reverse; - uint8_t highInclRowLow; - uint8_t highInclRowHigh; - }; + struct trackRebuildHelperCandidate; // Rebuild structs are defined in GPUTPCGMMergerRebuildInternal.h + struct trackRebuildHelper; + struct trackRebuildHelperRow; struct tmpSort { uint32_t x; @@ -113,6 +109,7 @@ class GPUTPCGMMerger : public GPUProcessor void SetMaxData(const GPUTrackingInOutPointers& io); void* SetPointersMerger(void* mem); void* SetPointersRefitScratch(void* mem); + void* SetPointersRebuild(void* mem); void* SetPointersOutput(void* mem); void* SetPointersOutputO2(void* mem); void* SetPointersOutputO2Clus(void* mem); @@ -136,7 +133,8 @@ class GPUTPCGMMerger : public GPUProcessor GPUhdi() uint32_t NMergedTrackClusters() const { return mMemory->nMergedTrackClusters; } GPUhdi() const GPUTPCGMMergedTrackHit* Clusters() const { return mClusters; } GPUhdi() GPUTPCGMMergedTrackHit* Clusters() { return mClusters; } - GPUhdi() trackCluster* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelperCandidate* ClusterCandidates() { return mClusterCandidates; } + GPUhdi() trackRebuildHelperRow* TrackRebuildHelperRow() { return mTrackRebuildHelperRow; } GPUhdi() trackRebuildHelper* TrackRebuildHelper() { return mTrackRebuildHelper; } GPUhdi() int32_t* HitWeights() { return mHitWeights; } GPUhdi() GPUAtomic(uint32_t) * ClusterAttachment() const { return mClusterAttachment; } @@ -173,6 +171,7 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() uint16_t MemoryResOutputO2Clus() const { return mMemoryResOutputO2Clus; } GPUd() uint16_t MemoryResOutputO2MC() const { return mMemoryResOutputO2MC; } GPUd() uint16_t MemoryResOutputO2Scratch() const { return mMemoryResOutputO2Scratch; } + GPUd() uint16_t MemoryResRebuild() const { return mMemoryResRebuild; } GPUd() int32_t RefitSectorTrack(GPUTPCGMSectorTrack& sectorTrack, const GPUTPCTrack* inTrack, float alpha, int32_t sector); GPUd() void SetTrackClusterT(GPUTPCGMSectorTrack& track, int32_t iSector, const GPUTPCTrack* sectorTr); @@ -210,6 +209,8 @@ class GPUTPCGMMerger : public GPUProcessor GPUd() void MergeLoopersInit(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersSort(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void MergeLoopersMain(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); + GPUd() uint32_t CandidateId(uint32_t clusterId, uint8_t sector, uint8_t row) const; // Id for trackRebuildHelperCandidate (index in the sector row) of a global cluster id + GPUd() uint32_t CandidateClusterId(trackRebuildHelperCandidate candidate, uint8_t row) const; // Global cluster id of a candidate with cluster GPUd() void PrepareHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread); GPUd() void ComputeHitWeights(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); GPUd() void ResolveHitWeights1(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, int32_t iteration); @@ -294,11 +295,13 @@ class GPUTPCGMMerger : public GPUProcessor uint16_t mMemoryResOutputO2Clus = (uint16_t)-1; uint16_t mMemoryResOutputO2MC = (uint16_t)-1; uint16_t mMemoryResOutputO2Scratch = (uint16_t)-1; + uint16_t mMemoryResRebuild = (uint16_t)-1; - int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) - GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks + int32_t mNSectorHits = 0; // Total number of incoming clusters (from sector tracks) + GPUTPCGMMergedTrack* mMergedTracks = nullptr; //* array of output merged tracks o2::MCCompLabel* mMergedTrackMC = nullptr; - trackCluster* mClusterCandidates = nullptr; + trackRebuildHelperCandidate* mClusterCandidates = nullptr; + trackRebuildHelperRow* mTrackRebuildHelperRow = nullptr; trackRebuildHelper* mTrackRebuildHelper = nullptr; int32_t* mHitWeights = nullptr; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx index 2ce927a5a5f7d..1472fa7984068 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerDump.cxx @@ -21,6 +21,7 @@ #include "GPUTPCTracker.h" #include "GPUTPCTrackParam.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUO2DataTypes.h" #include "GPUCommonMath.h" #include "GPUTPCTrackParam.h" @@ -273,14 +274,14 @@ void GPUTPCGMMerger::DumpInterpolatedHits(std::ostream& out) const out << "Track " << i << ":"; for (uint32_t j = 0; j < GPUTPCGeometry::NROWS; j++) { auto* candidates = &mClusterCandidates[(i * GPUTPCGeometry::NROWS + j) * Param().rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidates[0].id) { - out << " Row " << j << ": "; - if (candidates[0].best) { - out << " Best " << candidates[0].best << " - "; + if (!candidates[0].isEmpty()) { + out << " Row " << j << " (weight " << mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getWeight() << "): "; + if (mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest()) { + out << " Best " << mTrackRebuildHelperRow[i * GPUTPCGeometry::NROWS + j].getBest() << " - "; } for (uint32_t k = 0; k < Param().rec.tpc.rebuildTrackInFitClusterCandidates; k++) { - if (candidates[k].id) { - out << k << ": id " << candidates[k].id << " err " << candidates[k].error << " weight " << candidates[k].weight << " - "; + if (!candidates[k].isEmpty()) { + out << k << ": id " << (candidates[k].hasCluster() ? CandidateClusterId(candidates[k], j) + 2 : candidates[k].getValue()) << " err " << (float)candidates[k].error << " - "; } } out << "; "; diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h new file mode 100644 index 0000000000000..ec040469b0658 --- /dev/null +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerRebuildInternal.h @@ -0,0 +1,64 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// \file GPUTPCGMMergerRebuildInternal.h +/// \author David Rohr + +#ifndef GPUTPCGMMERGERREBUILDINTERNAL_H +#define GPUTPCGMMERGERREBUILDINTERNAL_H + +#include "GPUTPCGMMerger.h" +#include "GPUCommonMath.h" + +namespace o2::gpu +{ + +struct GPUTPCGMMerger::trackRebuildHelperCandidate { // Per track, row and candidate, the row is implicit + uint16_t idHigh, idLow; // 2 x 16 bit instead of uint32_t for a size of 6 bytes without packing, high part first. Combined value: 0: empty (in the first candidate: row not searched), 1: row searched but no candidate, otherwise (sector << 24) | (id + 2), with id the index of the cluster in its sector row (below 2^24 - 2) + CAMath::half_t error; + GPUdi() uint32_t getValue() const { return ((uint32_t)idHigh << 16) | idLow; } + GPUdi() void setValue(uint32_t v) + { + idHigh = v >> 16; + idLow = v & 0xFFFF; + } + GPUdi() bool isEmpty() const { return getValue() == 0; } + GPUdi() bool hasCluster() const { return getValue() >= 2; } + GPUdi() void setNoCandidate() { setValue(1); } + GPUdi() uint32_t getId() const { return (getValue() & 0xFFFFFF) - 2; } // Only with hasCluster(), convert with GPUTPCGMMerger::CandidateId / CandidateClusterId + GPUdi() uint8_t getSector() const { return idHigh >> 8; } // Only with hasCluster() + GPUdi() void set(uint32_t id, uint8_t sector, CAMath::half_t e) + { + setValue(((uint32_t)sector << 24) | (id + 2)); + error = e; + } +}; +static_assert(sizeof(GPUTPCGMMerger::trackRebuildHelperCandidate) == 6); + +struct GPUTPCGMMerger::trackRebuildHelper { // Per track + uint8_t reverse; + uint8_t highInclRowLow; + uint8_t highInclRowHigh; +}; + +struct GPUTPCGMMerger::trackRebuildHelperRow { // Per track and row: weight (lower 28 bits) and best (upper 4 bits) + uint32_t data; + enum : uint32_t { bestRowRemoved = 14, // Row removed (extrapolation reverted) + bestNoCandidate = 15 }; // No acceptable candidate + GPUdi() uint32_t getBest() const { return data >> 28; } // 0: undecided, 1 - 13: index of selected candidate + 1, or bestRowRemoved / bestNoCandidate + GPUdi() void setBest(uint32_t best) { data = (data & 0x0FFFFFFF) | (best << 28); } + GPUdi() int32_t getWeight() const { return data & 0x0FFFFFFF; } + GPUdi() int32_t setWeight(float w) { return (data = (data & 0xF0000000) | (w > 0.5f ? (w < 2147483648.f ? CAMath::Float2UIntReint(w) - 0x3F000000 : 0x0FFFFFFF) : 0)) & 0x0FFFFFFF; } +}; + +} // namespace o2::gpu + +#endif diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 46254350d1393..0e2a5e94b4291 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -23,6 +23,7 @@ #include "GPUTPCGMMergedTrack.h" #include "GPUTPCGMPolynomialField.h" #include "GPUTPCGMMerger.h" +#include "GPUTPCGMMergerRebuildInternal.h" #include "GPUTPCTracker.h" #include "GPUdEdx.h" #include "GPUParam.h" @@ -385,20 +386,20 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ inter.markInvalid(); float uncorrectedY = FindBestInterpolatedHit(merger, inter, sector, iExtrapolatedRow, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 4) ? adjacentLeg : 0, true); auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - bool found = candidate.id >= 2; + bool found = candidate.hasCluster(); if (found) { lastExtrapolateRow = iExtrapolatedRow; #if defined(DEBUG_REBUILD_MC) && !defined(GPUCA_GPUCODE) if (merger.MergedTrackMC() && merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth) { - int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(candidate.id - 2), merger.MergedTrackMC()[iTrk]); + int32_t labelCorrect = GPUTPCTrkLblSearch(merger.GetConstantMem()->ioPtrs.clustersNative->clustersMCTruth->getLabels(merger.CandidateClusterId(candidate, iExtrapolatedRow)), merger.MergedTrackMC()[iTrk]); CADEBUG(printf("\t%21sLabel correct: %d\n", "", labelCorrect)); } #endif float err2Y, err2Z, xx, yy, zz; - const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[candidate.id - 2]; + const ClusterNative& GPUrestrict() cl = merger.GetConstantMem()->ioPtrs.clustersNative->clustersLinear[merger.CandidateClusterId(candidate, iExtrapolatedRow)]; merger.GetConstantMem()->calibObjects.fastTransform->Transform(sector, iExtrapolatedRow, cl.getPad(), cl.getTime(), xx, yy, zz, mTOffset); if (prop.PropagateToXAlpha(xx, prop.GetAlpha(), inFlyDirection)) { - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; break; } @@ -417,12 +418,12 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ int32_t retValUpd = prop.Update(yy, zz, iExtrapolatedRow, param, clusterState, true, refit, err2Y, err2Z); CADEBUG(printf("\tExtrapol. Sec %2d Row %3d Fit Alpha %8.3f , X %8.3f - Y %8.3f, Z %8.3f - QPt %7.2f (%7.2f), SP %5.2f (%5.2f), DzDs %5.2f %16s --- Cov sY %8.3f sZ %8.3f sSP %8.3f sPt %8.3f - YPt %8.3f - FErr %d\n", sector, iExtrapolatedRow, prop.GetAlpha(), mX, mP[0], mP[1], mP[4], prop.GetQPt0(), mP[2], prop.GetSinPhi0(), mP[3], "", sqrtf(mC[0]), sqrtf(mC[2]), sqrtf(mC[5]), sqrtf(mC[14]), mC[10], retValUpd)); if (retValUpd == 0 && param.rec.tpc.rebuildTrackMaxQPtDrift > 0 && CAMath::Abs(mP[4] - qPtStart) > param.rec.tpc.rebuildTrackMaxQPtDrift * CAMath::Max(1.f, CAMath::Abs(qPtStart))) { // Track changed too much, probably following something else - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); revert = true; // The rejected cluster is already in the fit break; } if (retValUpd != 0) { - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iExtrapolatedRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); if (retValUpd < GPUTPCGMPropagator::updateErrorClusterRejected) { revert = true; break; @@ -466,8 +467,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } if (lastGoodRow != lastExtrapolateRow) { for (int32_t iRow = lastGoodRow + up; iRow != lastExtrapolateRow + up; iRow += up) { - auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + iRow) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0]; - candidate.best = -1; + merger.TrackRebuildHelperRow()[iTrk * GPUTPCGeometry::NROWS + iRow].setBest(GPUTPCGMMerger::trackRebuildHelperRow::bestRowRemoved); } } if (finalOutInFit && !(param.rec.tpc.disableMarkAdjacent & 8) && iExtrapolatedRow < GPUTPCGeometry::NROWS) { // Continue the looper following from where the extrapolation stopped @@ -614,9 +614,9 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger } else if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1) { if (track.NClusters() <= param.rec.tpc.rebuildTrackFallbackMaxNCl && inter.isValid()) { // Short track: keep the original cluster as candidate if the search found none, but it passes the interpolated chi2 cut auto& candidate = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + cluster.row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; - if (candidate.id < 2 && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { + if (!candidate.hasCluster() && !InterpolateReject(merger, prop, yy, zz, clusterState, &inter, err2Y, err2Z, deltaZ, cluster.sector, cluster.row, iTrk, 0)) { const float wY = mC[0] / (mC[0] + (float)inter.errorY), wZ = mC[2] / (mC[2] + (float)inter.errorZ); // Weight of the stored position in the interpolation - candidate = {.id = cluster.num + 2, .row = cluster.row, .sector = cluster.sector, .error = CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])), .weight = 0, .best = 0}; + candidate.set(merger.CandidateId(cluster.num, cluster.sector, cluster.row), cluster.sector, (CAMath::half_t)(CAMath::Square(yy - mP[0] - wY * ((float)inter.posY - mP[0])) + CAMath::Square(zz - mP[1] - wZ * ((float)inter.posZ + deltaZ - mP[1])))); } } } else if (!param.rec.tpc.mergerInterpolateErrors && !param.rec.tpc.rebuildTrackInFit) { @@ -798,7 +798,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre } int32_t nCandidates = 0; - while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].id > 1) { + while (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates && merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + nCandidates].hasCluster()) { nCandidates++; } if (CAMath::Abs(uncorrectedY) <= rowData.getTPCMaxY()) { @@ -854,7 +854,7 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre if (ok) { int32_t insert = nCandidates; for (int32_t c = 0; c < nCandidates; c++) { - if (err < merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { + if (err < (float)merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c].error) { insert = c; break; } @@ -863,17 +863,17 @@ GPUdii() float GPUTPCGMTrackParam::FindBestInterpolatedHit(GPUTPCGMMerger& GPUre for (int32_t c = CAMath::Min(nCandidates, param.rec.tpc.rebuildTrackInFitClusterCandidates - 1); c > insert; c--) { merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c] = merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + c - 1]; } // TODO: Downweight non-interpolated candidates - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert] = {.id = (uint32_t)(idOffset + ids[ih] + 2), .row = row, .sector = sector, .error = err, .weight = 0, .best = 0}; + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + insert].set(merger.CandidateId(idOffset + ids[ih], sector, row), sector, (CAMath::half_t)err); nCandidates += (nCandidates < param.rec.tpc.rebuildTrackInFitClusterCandidates); } } } } } - CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].id > 1) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, dbgCand[dbgi].id - 2, dbgCand[dbgi].error); else break; }); + CADEBUG(const auto* dbgCand = &merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates]; for (int dbgi = 0; dbgi < nCandidates; dbgi++) { if (dbgCand[dbgi].hasCluster()) printf("\t\t\tiTrk %d Row %d Candidate %d hit %d err %f\n", iTrk, (int)row, dbgi, merger.CandidateClusterId(dbgCand[dbgi], row), dbgCand[dbgi].error); else break; }); } if (nCandidates == 0 && !interOnly) { // TODO: remove the interOnly here when rebuilding is fully working - merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].id = 1; + merger.ClusterCandidates()[(iTrk * GPUTPCGeometry::NROWS + row) * param.rec.tpc.rebuildTrackInFitClusterCandidates + 0].setNoCandidate(); } } else if (doAttach && (!interOnly || inter.isValid())) { const float y = interOnly ? (float)inter.posY : mP[0], z = interOnly ? (float)inter.posZ + deltaZ : mP[1]; From b414331b9c7c6ac0f0e627af9e341c46a6cffdf6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 19 Nov 2026 14:02:29 +0100 Subject: [PATCH 54/54] GPU TPC: Faster implementation for search adjacent clusters in missing rows --- GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h | 3 +-- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 21 ++++++++++++++++--- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h index 7dd7d31a4f9cb..510583f01bc93 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMMergerTypes.h @@ -40,8 +40,7 @@ struct InterpolationErrorHit { GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A errorY, errorZ; GPUdi() bool isValid() const { return errorY >= (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)0; } GPUdi() void markInvalid() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-1; } - GPUdi() bool isAdjacentDone() const { return errorY == (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } - GPUdi() void markAdjacentDone() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } // Adjacent clusters were searched at the interpolated position of this entry, the entry is invalid afterwards + GPUdi() void markAdjacentSearch() { errorY = (GPUCA_PAR_MERGER_INTERPOLATION_ERROR_TYPE_A)-2; } // Invalidates an entry the interpolation rejection did not use, such that the last way searches adjacent clusters for the hit (distinct from the -1 of markInvalid) }; struct InterpolationErrors { diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 0e2a5e94b4291..b935ac049c980 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -89,6 +89,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ ConstrainZ(merger, clusters[0].sector, CAMath::Max(t0, tN), CAMath::Min(t0, tN)); } int32_t ihitStart = 0; + int32_t ihitStartInterpolateReject = 0; // First hit of the interpolation rejection way, the hits beyond are not covered by its search for adjacent clusters int32_t interpolatedStart = 0; float covYYUpd = 0.f; float deltaZ = 0.f; @@ -127,6 +128,9 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ const bool inFlyDirection = iWay & 1; const int32_t wayDirection = (iWay & 1) ? -1 : 1; + if (param.rec.tpc.mergerInterpolateErrors && iWay == nWays - 2) { + ihitStartInterpolateReject = ihitStart; + } for (int32_t ihit = ihitStart, interpolationIndex = interpolatedStart; ihit >= 0 && ihit < maxN; ihit += wayDirection) { if (lastIndexRow != 255) { // One index per pad row (2 if the track has clusters of 2 sectors in the row), counted also for hits skipped below, such that all ways use the same index for a hit interpolationIndex += (clusters[ihit].row == lastIndexRow ? (int32_t)(clusters[ihit].sector != lastIndexSector) : CAMath::Abs(clusters[ihit].row - lastIndexRow)) * wayDirection; @@ -211,6 +215,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagHighIncl); nMissedHard++; NTolerated++; + MarkAdjacentSearch(param, iWay, inter); if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, true, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } @@ -224,14 +229,15 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && param.rec.tpc.rebuildTrackMaxNonIntCov > 0 && !(param.rec.tpc.disableRebuildAttachment & 4)) { InterpolateMissingRows(merger, interpolation, clusters, ihit, interpolationIndex, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); } + MarkAdjacentSearch(param, iWay, inter); fitBroken = true; break; // bad chi2 for the whole track, stop the fit } float uncorrectedY = -1e6f; if (param.rec.tpc.rebuildTrackInFit && iWay == wayRebuildStart + 1 && !(param.rec.tpc.disableRebuildAttachment & 1)) { - uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps - } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && !inter.isAdjacentDone()) : allowChangeClusters)) { + uncorrectedY = FindBestInterpolatedHit(merger, inter, cluster.sector, cluster.row, deltaZ, sumInvSqrtCharge, nAvgCharge, prop, iTrk, false, adjacentInRebuild && !(param.rec.tpc.disableMarkAdjacent & 1) ? adjacentLeg : 0); // TODO: Check whether we do not need to use track Y for some rejection steps + } else if (adjacentInRebuild ? iWay == wayRebuildStart + 1 : (param.rec.tpc.mergerInterpolateErrors ? (finalFit && (retryAttempt || !inter.isValid() || (ihit - ihitStartInterpolateReject) * wayDirection > 0 || (param.rec.tpc.disableMarkAdjacent & 1))) : allowChangeClusters)) { // With interpolation rejection: in the last way all hits InterpolateReject did not search: in a retry, beyond its first hit, or without interpolated position (also invalidated where it skipped a hit) uncorrectedY = FindAdjacentClusters(merger, cluster.sector, cluster.row, iTrk, track.Leg() == 0, prop); } if (param.rec.tpc.rejectEdgeClustersInTrackFit && uncorrectedY <= -1e6f) { @@ -249,6 +255,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ if (currentClusterStatus) { nMissed++; nMissedHard++; + MarkAdjacentSearch(param, iWay, inter); continue; } @@ -286,6 +293,7 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ } // TODO: We can perhaps break here, if we pick up remaining rows } + MarkAdjacentSearch(param, iWay, inter); continue; } } @@ -526,6 +534,13 @@ GPUd() bool GPUTPCGMTrackParam::Fit(GPUTPCGMMerger& GPUrestrict() merger, int32_ return true; } +GPUdi() void GPUTPCGMTrackParam::MarkAdjacentSearch(const GPUParam& GPUrestrict() param, int32_t iWay, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter) +{ + if (param.rec.tpc.mergerInterpolateErrors && iWay == param.rec.tpc.nWays - 2) { // Hit not passed to InterpolateReject, search adjacent clusters in the last way + inter.markAdjacentSearch(); + } +} + GPUdii() void GPUTPCGMTrackParam::FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& GPUrestrict() track, GPUTPCGMPropagator& GPUrestrict() prop, const bool inFlyDirection, GPUTPCGMMerger& GPUrestrict() merger, uint8_t* GPUrestrict() dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& GPUrestrict() inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge) { if (CAMath::Abs(mP[2]) > constants::MAX_SIN_PHI || CAMath::Abs(mP[0]) > CAMath::Abs(mX) * CAMath::Tan(GPUTPCGeometry::kSectAngle() / 2.f)) { @@ -586,6 +601,7 @@ GPUdii() int32_t GPUTPCGMTrackParam::FitHit(GPUTPCGMMerger& GPUrestrict() merger float threshold = 3.f + (lastUpdateX >= 0 ? (CAMath::Abs(mX - lastUpdateX) / 2) : 0.f); if (mNDF > (int32_t)param.rec.tpc.mergerNonInterpolateRejectMinNDF && (CAMath::Abs(yy - mP[0]) > threshold || CAMath::Abs(zz - mP[1]) > threshold)) { retValUpd = GPUTPCGMPropagator::updateErrorClusterRejectedDistance; + MarkAdjacentSearch(param, iWay, inter); if (param.rec.tpc.rebuildTrackInFit && (finalOutInFit || iWay < wayRebuildStart + 2)) { MarkClusters(clusters, ihitMergeFirst, ihit, wayDirection, GPUTPCGMMergedTrackHit::flagRejectDistance); // Will enable rejectChi2 in further rounds } @@ -726,7 +742,6 @@ GPUd() int32_t GPUTPCGMTrackParam::InterpolateReject(const GPUTPCGMMerger& GPUre param.GetClusterErrors2(sector, row, ImP1, mP[2], mP[3], -1.f, 0.f, 0.f, clErr2Y, clErr2Z); // TODO: Use correct time/avgCharge FindAdjacentClustersUncorrectedPos(merger, sector, row, iTrk, doAttach == 2, uncorrectedY, uncorrectedZ, ImP1, clErr2Y + CAMath::Abs(ImC0), clErr2Z + CAMath::Abs(ImC2)); } - inter->markAdjacentDone(); } if (GPUTPCGMPropagator::RejectCluster(chi2Y * param.rec.tpc.clusterRejectChi2TolleranceY, chi2Z * param.rec.tpc.clusterRejectChi2TolleranceZ, clusterState)) { // TODO: Relative Pt resolution decreases slightly, why? // printf("Reject Cluster chiy2 %f chiz2 %f (Pos Y: %f - %f %f ; Pos Z: %f - %f %f)\n", chi2Y, chi2Z, posY, mP[0], (float)inter->posY, posZ, mP[1], (float)inter->posZ + deltaZ); diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h index 665c2da4ef14d..31299b2fb8a72 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.h @@ -160,6 +160,7 @@ class GPUTPCGMTrackParam GPUd() int32_t FitHit(GPUTPCGMMerger& merger, const int32_t iTrk, const GPUTPCGMMergedTrack& track, const float xx, const float yy, const float zz, const uint8_t clusterState, const float clAlpha, const int32_t iWay, const bool inFlyDirection, float& deltaZ, float& lastUpdateX, GPUTPCGMMergedTrackHit* clusters, GPUTPCGMPropagator& prop, gputpcgmmergertypes::InterpolationErrorHit& inter, GPUdEdx& dEdx, GPUdEdx& dEdxAlt, float& sumInvSqrtCharge, int32_t& nAvgCharge, const int32_t ihit, const int32_t ihitMergeFirst, const bool allowChangeClusters, const bool refit, const bool finalFit, int32_t& nMissed, int32_t& nMissedHard, float uncorrectedY, bool retryAttempt); GPUd() void FitAddRow(const int32_t iRow, const uint8_t sector, const int32_t iTrk, const GPUTPCGMMergedTrack& track, GPUTPCGMPropagator& prop, const bool inFlyDirection, GPUTPCGMMerger& merger, uint8_t* dEdxSubThresholdRow, const bool dodEdx, const bool doAttach, const uint8_t doInterpolate, gputpcgmmergertypes::InterpolationErrorHit& inter, const float deltaZ, const float sumInvSqrtCharge, const int32_t nAvgCharge); GPUd() void HandleCrossCE(const GPUParam& param, const uint8_t sector, const uint8_t& lastSector); + GPUd() static void MarkAdjacentSearch(const GPUParam& param, int32_t iWay, gputpcgmmergertypes::InterpolationErrorHit& inter); // For hits the interpolation rejection way skips template GPUd() static void RefitTrack(GPUTPCGMMergedTrack& track, int32_t iTrk, GPUTPCGMMerger& merger); GPUd() void MoveToReference(GPUTPCGMPropagator& prop, const GPUParam& param, float& alpha);