Skip to content

Commit e486093

Browse files
fwyzardfwyzard
committed
Implement edmtest::GenericCloner
This EDProducer will clone all the event products declared by its configuration, using their ROOT dictionaries. Refactor common functionality with GenericConsumer.
1 parent c2c635b commit e486093

File tree

6 files changed

+367
-82
lines changed

6 files changed

+367
-82
lines changed

‎FWCore/Modules/src/GenericConsumer.cc

Lines changed: 10 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -40,80 +40,7 @@
4040
#include "FWCore/ParameterSet/interface/ParameterDescriptionNode.h"
4141
#include "FWCore/ParameterSet/interface/ParameterSet.h"
4242
#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
43-
44-
namespace {
45-
struct ProductBranch {
46-
public:
47-
ProductBranch(std::string const& label) {
48-
static const char kSeparator = '_';
49-
static const char kWildcard = '*';
50-
static const std::regex kAny{".*"};
51-
52-
// wildcard
53-
if (label == kWildcard) {
54-
type_ = kAny;
55-
moduleLabel_ = kAny;
56-
productInstanceName_ = kAny;
57-
processName_ = kAny;
58-
return;
59-
}
60-
61-
int fields = std::count(label.begin(), label.end(), kSeparator) + 1;
62-
if (fields == 1) {
63-
// convert the module label into a regular expression
64-
type_ = kAny;
65-
moduleLabel_ = glob_to_regex(label);
66-
productInstanceName_ = kAny;
67-
processName_ = kAny;
68-
} else if (fields == 4) {
69-
// split the branch name into <product type>_<module label>_<instance name>_<process name>
70-
// and convert the glob expressions into regular expressions
71-
size_t first = 0, last = 0;
72-
last = label.find(kSeparator, first);
73-
type_ = glob_to_regex(label.substr(first, last - first));
74-
first = last + 1;
75-
last = label.find(kSeparator, first);
76-
moduleLabel_ = glob_to_regex(label.substr(first, last - first));
77-
first = last + 1;
78-
last = label.find(kSeparator, first);
79-
productInstanceName_ = glob_to_regex(label.substr(first, last - first));
80-
first = last + 1;
81-
last = label.find(kSeparator, first);
82-
processName_ = glob_to_regex(label.substr(first, last - first));
83-
} else {
84-
// invalid input
85-
throw edm::Exception(edm::errors::Configuration) << "Invalid module label or branch name: \"" << label << "\"";
86-
}
87-
}
88-
89-
bool match(edm::BranchDescription const& branch) const {
90-
return (std::regex_match(branch.friendlyClassName(), type_) and
91-
std::regex_match(branch.moduleLabel(), moduleLabel_) and
92-
std::regex_match(branch.productInstanceName(), productInstanceName_) and
93-
std::regex_match(branch.processName(), processName_));
94-
}
95-
96-
private:
97-
static std::regex glob_to_regex(std::string pattern) {
98-
boost::replace_all(pattern, "*", ".*");
99-
boost::replace_all(pattern, "?", ".");
100-
return std::regex(pattern);
101-
}
102-
103-
std::regex type_;
104-
std::regex moduleLabel_;
105-
std::regex productInstanceName_;
106-
std::regex processName_;
107-
};
108-
109-
std::vector<ProductBranch> make_patterns(std::vector<std::string> const& labels) {
110-
std::vector<ProductBranch> patterns;
111-
patterns.reserve(labels.size());
112-
for (auto const& label : labels)
113-
patterns.emplace_back(label);
114-
return patterns;
115-
}
116-
} // namespace
43+
#include "FWCore/Utilities/interface/BranchPattern.h"
11744

11845
namespace edm {
11946
class GenericConsumer : public edm::global::EDAnalyzer<> {
@@ -126,19 +53,20 @@ namespace edm {
12653
static void fillDescriptions(ConfigurationDescriptions& descriptions);
12754

12855
private:
129-
std::vector<ProductBranch> eventProducts_;
130-
std::vector<ProductBranch> lumiProducts_;
131-
std::vector<ProductBranch> runProducts_;
132-
std::vector<ProductBranch> processProducts_;
56+
std::vector<edm::BranchPattern> eventProducts_;
57+
std::vector<edm::BranchPattern> lumiProducts_;
58+
std::vector<edm::BranchPattern> runProducts_;
59+
std::vector<edm::BranchPattern> processProducts_;
13360
std::string label_;
13461
bool verbose_;
13562
};
13663

13764
GenericConsumer::GenericConsumer(ParameterSet const& config)
138-
: eventProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("eventProducts"))),
139-
lumiProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("lumiProducts"))),
140-
runProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("runProducts"))),
141-
processProducts_(make_patterns(config.getUntrackedParameter<std::vector<std::string>>("processProducts"))),
65+
: eventProducts_(edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("eventProducts"))),
66+
lumiProducts_(edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("lumiProducts"))),
67+
runProducts_(edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("runProducts"))),
68+
processProducts_(
69+
edm::branchPatterns(config.getUntrackedParameter<std::vector<std::string>>("processProducts"))),
14270
label_(config.getParameter<std::string>("@module_label")),
14371
verbose_(config.getUntrackedParameter<bool>("verbose")) {
14472
callWhenNewProductsRegistered([this](edm::BranchDescription const& branch) {

‎FWCore/TestModules/README.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,14 @@ product read from the `Event`.
2424
Together `edmtest::EventIDProducer` and `edmtest::EventIDValidator` can be used
2525
to validate that an object produced in a given event is being read back in the
2626
same event.
27+
28+
29+
## `edmtest::GenericCloner`
30+
31+
This module will clone all the event products declared by its configuration,
32+
using their ROOT dictionaries.
33+
The products can be specified either as module labels (_e.g._ `<module label>`)
34+
or as branch names (_e.g._ `<product type>_<module label>_<instance name>_<process name>`).
35+
Glob expressions (`?` and `*`) are supported in module labels and within the
36+
individual fields of branch names, similar to an `OutputModule`'s `keep`
37+
statements.

‎FWCore/TestModules/plugins/GenericCloner.cc

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*
2+
* This EDProducer will clone all the event products declared by its configuration, using their ROOT dictionaries.
3+
*
4+
* The products can be specified either as module labels (e.g. "<module label>") or as branch names (e.g.
5+
* "<product type>_<module label>_<instance name>_<process name>").
6+
*
7+
* If a module label is used, no underscore ("_") must be present; this module will clone all the products produced by
8+
* that module, including those produced by the Transformer functionality (such as the implicitly copied-to-host
9+
* products in case of Alpaka-based modules).
10+
* If a branch name is used, all four fields must be present, separated by underscores; this module will clone only on
11+
* the matching product(s).
12+
*
13+
* Glob expressions ("?" and "*") are supported in module labels and within the individual fields of branch names,
14+
* similar to an OutputModule's "keep" statements.
15+
* Use "*" to clone all products.
16+
*
17+
* For example, in the case of Alpaka-based modules running on a device, using
18+
*
19+
* eventProducts = cms.untracked.vstring( "module" )
20+
*
21+
* will cause "module" to run, along with automatic copy of its device products to the host, and will attempt to clone
22+
* all device and host products.
23+
* To clone only the host product, the branch can be specified explicitly with
24+
*
25+
* eventProducts = cms.untracked.vstring( "HostProductType_module_*_*" )
26+
*
27+
* .
28+
*/
29+
30+
#include <cstring>
31+
#include <memory>
32+
#include <string>
33+
#include <string_view>
34+
#include <utility>
35+
#include <vector>
36+
37+
#include <TBufferFile.h>
38+
39+
#include "DataFormats/Provenance/interface/BranchDescription.h"
40+
#include "FWCore/Framework/interface/Event.h"
41+
#include "FWCore/Framework/interface/GenericHandle.h"
42+
#include "FWCore/Framework/interface/GenericProduct.h"
43+
#include "FWCore/Framework/interface/global/EDProducer.h"
44+
#include "FWCore/MessageLogger/interface/MessageLogger.h"
45+
#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
46+
#include "FWCore/ParameterSet/interface/ParameterDescriptionNode.h"
47+
#include "FWCore/ParameterSet/interface/ParameterSet.h"
48+
#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
49+
#include "FWCore/Reflection/interface/ObjectWithDict.h"
50+
#include "FWCore/Utilities/interface/BranchPattern.h"
51+
52+
namespace edmtest {
53+
54+
class GenericCloner : public edm::global::EDProducer<> {
55+
public:
56+
explicit GenericCloner(edm::ParameterSet const&);
57+
~GenericCloner() override = default;
58+
59+
void produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const override;
60+
61+
static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
62+
63+
private:
64+
struct Entry {
65+
edm::TypeWithDict objectType_;
66+
edm::TypeWithDict wrappedType_;
67+
edm::EDGetToken getToken_;
68+
edm::EDPutToken putToken_;
69+
};
70+
71+
std::vector<edm::BranchPattern> eventPatterns_;
72+
std::vector<Entry> eventProducts_;
73+
std::string label_;
74+
bool verbose_;
75+
};
76+
77+
GenericCloner::GenericCloner(edm::ParameterSet const& config)
78+
: eventPatterns_(edm::branchPatterns(config.getParameter<std::vector<std::string>>("eventProducts"))),
79+
label_(config.getParameter<std::string>("@module_label")),
80+
verbose_(config.getUntrackedParameter<bool>("verbose")) {
81+
eventProducts_.reserve(eventPatterns_.size());
82+
83+
callWhenNewProductsRegistered([this](edm::BranchDescription const& branch) {
84+
static const std::string_view kPathStatus("edm::PathStatus");
85+
static const std::string_view kEndPathStatus("edm::EndPathStatus");
86+
87+
switch (branch.branchType()) {
88+
case edm::InEvent:
89+
if (branch.className() == kPathStatus or branch.className() == kEndPathStatus)
90+
return;
91+
for (auto& pattern : eventPatterns_)
92+
if (pattern.match(branch)) {
93+
Entry product;
94+
product.objectType_ = branch.unwrappedType();
95+
product.wrappedType_ = branch.wrappedType();
96+
// TODO move this to EDConsumerBase::consumes() ?
97+
product.getToken_ = this->consumes(
98+
edm::TypeToGet{branch.unwrappedTypeID(), edm::PRODUCT_TYPE},
99+
edm::InputTag{branch.moduleLabel(), branch.productInstanceName(), branch.processName()});
100+
product.putToken_ = this->produces(branch.unwrappedTypeID(), branch.productInstanceName());
101+
eventProducts_.push_back(product);
102+
103+
if (verbose_) {
104+
edm::LogInfo("GenericCloner")
105+
<< label_ << " will clone Event product " << branch.friendlyClassName() << '_'
106+
<< branch.moduleLabel() << '_' << branch.productInstanceName() << '_' << branch.processName();
107+
}
108+
break;
109+
}
110+
break;
111+
112+
case edm::InLumi:
113+
case edm::InRun:
114+
case edm::InProcess:
115+
// lumi, run and process products are not supported
116+
break;
117+
118+
default:
119+
throw edm::Exception(edm::errors::LogicError)
120+
<< "Unexpected branch type " << branch.branchType() << "\nPlease contact a Framework developer\n";
121+
}
122+
});
123+
}
124+
125+
void GenericCloner::produce(edm::StreamID /*unused*/, edm::Event& event, edm::EventSetup const& /*unused*/) const {
126+
for (auto& product : eventProducts_) {
127+
edm::GenericHandle handle(product.objectType_);
128+
event.getByToken(product.getToken_, handle);
129+
edm::ObjectWithDict const* object = handle.product();
130+
131+
TBufferFile send_buffer(TBuffer::kWrite);
132+
send_buffer.WriteObjectAny(object->address(), product.objectType_.getClass(), false);
133+
int size = send_buffer.Length();
134+
135+
TBufferFile recv_buffer(TBuffer::kRead, size);
136+
std::memcpy(recv_buffer.Buffer(), send_buffer.Buffer(), size);
137+
138+
void* clone_ptr = reinterpret_cast<void*>(recv_buffer.ReadObjectAny(product.objectType_.getClass()));
139+
auto clone = std::make_unique<edm::GenericProduct>();
140+
clone->object_ = edm::ObjectWithDict(product.objectType_, clone_ptr);
141+
clone->wrappedType_ = product.wrappedType_;
142+
143+
// specialise Event::put for GenericProduct
144+
event.put(product.putToken_, std::move(clone));
145+
}
146+
}
147+
148+
void GenericCloner::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
149+
descriptions.setComment(
150+
R"(This EDProducer will clone all the event products declared by its configuration, using their ROOT dictionaries.
151+
152+
The products can be specified either as module labels (e.g. "<module label>") or as branch names (e.g. "<product type>_<module label>_<instance name>_<process name>").
153+
If a module label is used, no underscore ("_") must be present; this module will clone all the products produced by that module, including those produced by the Transformer functionality (such as the implicitly copied-to-host products in case of Alpaka-based modules).
154+
If a branch name is used, all four fields must be present, separated by underscores; this module will clone only on the matching product(s).
155+
156+
Glob expressions ("?" and "*") are supported in module labels and within the individual fields of branch names, similar to an OutputModule's "keep" statements.
157+
Use "*" to clone all products.
158+
159+
For example, in the case of Alpaka-based modules running on a device, using
160+
161+
eventProducts = cms.untracked.vstring( "module" )
162+
163+
will cause "module" to run, along with automatic copy of its device products to the host, and will attempt to clone all device and host products.
164+
To clone only the host product, the branch can be specified explicitly with
165+
166+
eventProducts = cms.untracked.vstring( "HostProductType_module_*_*" )
167+
168+
.)");
169+
170+
edm::ParameterSetDescription desc;
171+
desc.add<std::vector<std::string>>("eventProducts", {})
172+
->setComment("List of modules or branches whose event products will be cloned.");
173+
desc.addUntracked<bool>("verbose", false)
174+
->setComment("Print the branch names of the products that will be cloned.");
175+
descriptions.addWithDefaultLabel(desc);
176+
}
177+
178+
} // namespace edmtest
179+
180+
#include "FWCore/Framework/interface/MakerMacros.h"
181+
DEFINE_FWK_MODULE(edmtest::GenericCloner);

‎FWCore/TestModules/test/BuildFile.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
<test name="TestFWCoreModulesEventIDValidator" command="cmsRun ${LOCALTOP}/src/FWCore/TestModules/test/testEventIDValidator_cfg.py"/>
2+
3+
<test name="TestFWCoreModulesGenericCloner" command="cmsRun ${LOCALTOP}/src/FWCore/TestModules/test/testGenericCloner_cfg.py"/>

‎FWCore/TestModules/test/testGenericCloner_cfg.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import FWCore.ParameterSet.Config as cms
2+
3+
process = cms.Process("TEST")
4+
5+
process.load("FWCore.MessageService.MessageLogger_cfi")
6+
process.MessageLogger.cerr.INFO.limit = 10000000
7+
8+
process.options.numberOfThreads = 4
9+
process.options.numberOfStreams = 4
10+
11+
process.source = cms.Source("EmptySource")
12+
process.maxEvents.input = 10
13+
14+
process.eventIds = cms.EDProducer("edmtest::EventIDProducer")
15+
16+
process.cloneByLabel = cms.EDProducer("edmtest::GenericCloner",
17+
eventProducts = cms.vstring("eventIds"),
18+
verbose = cms.untracked.bool(True)
19+
)
20+
21+
process.cloneByBranch = cms.EDProducer("edmtest::GenericCloner",
22+
eventProducts = cms.vstring("*_eventIds__TEST"),
23+
verbose = cms.untracked.bool(True)
24+
)
25+
26+
process.validateByLabel = cms.EDAnalyzer("edmtest::EventIDValidator",
27+
source = cms.untracked.InputTag('cloneByLabel')
28+
)
29+
30+
process.validateByBranch = cms.EDAnalyzer("edmtest::EventIDValidator",
31+
source = cms.untracked.InputTag('cloneByBranch')
32+
)
33+
34+
process.task = cms.Task(process.eventIds, process.cloneByLabel, process.cloneByBranch)
35+
36+
process.path = cms.Path(process.validateByLabel + process.validateByBranch, process.task)

0 commit comments

Comments
 (0)