This commit is contained in:
coco
2026-07-03 15:56:07 +08:00
commit caef23209c
5767 changed files with 1004268 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
# binjr-adapter-rrd4j
[![Maven Central](https://img.shields.io/maven-central/v/eu.binjr/binjr-adapter-rrd4j.svg?label=Maven%20Central&style=flat-square)](https://search.maven.org/search?q=g:%22eu.binjr%22%20AND%20a:%22binjr-adapter-rrd4j%22)
This module implements a DataAdapter capable of consuming data from Round Robin Database files.
It can read files produced by the following applications:
* [RRD4J](https://github.com/rrd4j/rrd4j), binary files (*.rrd) and XML dumps (*.xml)
* [RRDTool](https://oss.oetiker.ch/rrdtool/index.en.html), binary files (*.rrd) and XML dumps (*.xml)
+32
View File
@@ -0,0 +1,32 @@
/*
* Copyright 2018-2022 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
dependencies {
compileOnly project(':binjr-core')
implementation 'org.rrd4j:rrd4j:3.9'
}
jar {
manifest {
attributes(
'Specification-Title': project.name,
'Specification-Version': project.version,
'Implementation-Title': project.name,
'Implementation-Version': project.version,
'Build-Number': BINJR_BUILD_NUMBER
)
}
}
@@ -0,0 +1,40 @@
/*
* Copyright 2020 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.rrd4j.adapters;
/**
* RRD4J backend types.
*/
public enum Rrd4jBackendType {
/**
* Backend based on the java.io.* package. RRD data is stored in files on the disk
*/
FILE,
/**
* Backend based on the java.io.* package. RRD data is stored in files on the disk.
* This backend is "safe". Being safe means that RRD files can be safely shared between several JVM's.
*/
SAFE,
/**
* Backend based on the java.nio.* package. RRD data is stored in files on the disk
*/
NIO,
/**
* Memory-oriented backend. RRD data is stored in memory, it gets lost as soon as JVM exits.
*/
MEMORY;
}
@@ -0,0 +1,272 @@
/*
* Copyright 2018-2022 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.rrd4j.adapters;
import eu.binjr.common.javafx.controls.TimeRange;
import eu.binjr.common.logging.Logger;
import eu.binjr.core.data.adapters.BaseDataAdapter;
import eu.binjr.core.data.adapters.SourceBinding;
import eu.binjr.core.data.adapters.TimeSeriesBinding;
import eu.binjr.core.data.exceptions.DataAdapterException;
import eu.binjr.core.data.exceptions.FetchingDataFromAdapterException;
import eu.binjr.core.data.timeseries.DoubleTimeSeriesProcessor;
import eu.binjr.core.data.timeseries.TimeSeriesProcessor;
import eu.binjr.core.data.workspace.TimeSeriesInfo;
import eu.binjr.core.preferences.UserPreferences;
import javafx.scene.chart.XYChart;
import javafx.scene.control.TreeItem;
import org.eclipse.fx.ui.controls.tree.FilterableTreeItem;
import org.rrd4j.ConsolFun;
import org.rrd4j.core.*;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Collectors;
/**
* A {@link eu.binjr.core.data.adapters.DataAdapter} implementation capable of consuming data
* from Round Robin Database files.
*
* @author Frederic Thevenet
*/
public class Rrd4jFileAdapter extends BaseDataAdapter<Double> {
private static final Logger logger = Logger.create(Rrd4jFileAdapter.class);
private final Rrd4jFileAdapterPreferences prefs = (Rrd4jFileAdapterPreferences) this.getAdapterInfo().getPreferences();
private final Map<Path, RrdDb> rrdDbMap = new HashMap<>();
private List<Path> rrdPaths;
private List<Path> tempPathToCollect = new ArrayList<>();
/**
* Initialises a new instance of the {@link Rrd4jFileAdapter} class.
*/
public Rrd4jFileAdapter() {
this(new ArrayList<Path>());
}
/**
* Initialises a new instance of the {@link Rrd4jFileAdapter} class from the provided list of {@link Path}
*
* @param rrdPath a list of {@link Path} to be mounted by the adapter.
*/
public Rrd4jFileAdapter(List<Path> rrdPath) {
this.rrdPaths = rrdPath;
}
@Override
public FilterableTreeItem<SourceBinding> getBindingTree() throws DataAdapterException {
FilterableTreeItem<SourceBinding> tree = new FilterableTreeItem<>(
new TimeSeriesBinding.Builder()
.withLabel(getSourceName())
.withPath("/")
.withAdapter(this)
.build());
for (Path rrdPath : rrdPaths) {
try {
String rrdFileName = rrdPath.getFileName().toString();
FilterableTreeItem<SourceBinding> rrdNode = new FilterableTreeItem<>(
new TimeSeriesBinding.Builder()
.withLabel(rrdFileName)
.withPath(rrdFileName)
.withParent(tree.getValue())
.withAdapter(this)
.build());
RrdDb rrd = openRrdDb(rrdPath);
rrdDbMap.put(rrdPath, rrd);
for (ConsolFun consolFun : Arrays.stream(rrd.getRrdDef().getArcDefs())
.map(ArcDef::getConsolFun)
.collect(Collectors.toSet())) {
FilterableTreeItem<SourceBinding> consolFunNode = new FilterableTreeItem<>(new TimeSeriesBinding.Builder()
.withLabel(consolFun.toString())
.withPath(rrdPath.resolve(consolFun.toString()).toString())
.withParent(rrdNode.getValue())
.withAdapter(this)
.build());
rrdNode.getInternalChildren().add(consolFunNode);
for (String ds : rrd.getDsNames()) {
consolFunNode.getInternalChildren().add(new TreeItem<>(new TimeSeriesBinding.Builder()
.withLabel(ds)
.withPath(consolFunNode.getValue().getPath())
.withParent(consolFunNode.getValue())
.withAdapter(this)
.build()));
}
}
tree.getInternalChildren().add(rrdNode);
} catch (IOException e) {
throw new DataAdapterException("Failed to open rrd db", e);
}
}
return tree;
}
@Override
public TimeRange getInitialTimeRange(String path, List<TimeSeriesInfo<Double>> seriesInfo) throws DataAdapterException {
if (this.isClosed()) {
throw new IllegalStateException("An attempt was made to fetch data from a closed adapter");
}
Path dsPath = Path.of(path);
try {
var end = Instant.ofEpochSecond(rrdDbMap.get(dsPath.getParent()).getLastArchiveUpdateTime()).atZone(getTimeZoneId());
return TimeRange.of(end.minusHours(24), end);
} catch (IOException e) {
throw new FetchingDataFromAdapterException("IO Error while retrieving last update from rrd db", e);
}
}
@Override
public Map<TimeSeriesInfo<Double>, TimeSeriesProcessor<Double>> fetchData(String path, Instant begin, Instant
end, List<TimeSeriesInfo<Double>> seriesInfo, boolean bypassCache)
throws DataAdapterException {
if (this.isClosed()) {
throw new IllegalStateException("An attempt was made to fetch data from a closed adapter");
}
Path dsPath = Path.of(path);
try {
FetchRequest request = rrdDbMap.get(dsPath.getParent()).createFetchRequest(
ConsolFun.valueOf(dsPath.getFileName().toString()),
begin.getEpochSecond(),
end.getEpochSecond());
request.setFilter(seriesInfo.stream().map(s -> s.getBinding().getLabel()).toArray(String[]::new));
FetchData data = request.fetchData();
Map<TimeSeriesInfo<Double>, TimeSeriesProcessor<Double>> series = new HashMap<>();
for (int i = 0; i < data.getRowCount(); i++) {
ZonedDateTime timeStamp = Instant.ofEpochSecond(data.getTimestamps()[i]).atZone(getTimeZoneId());
for (TimeSeriesInfo<Double> info : seriesInfo) {
Double val = data.getValues(info.getBinding().getLabel())[i];
XYChart.Data<ZonedDateTime, Double> point = new XYChart.Data<>(timeStamp, val);
TimeSeriesProcessor<Double> seriesProcessor =
series.computeIfAbsent(info, k -> new DoubleTimeSeriesProcessor());
seriesProcessor.addSample(point);
}
}
logger.trace(() -> String.format("Built %d series with %d samples each (%d total samples)",
seriesInfo.size(),
data.getRowCount(),
seriesInfo.size() * data.getRowCount()));
return series;
} catch (IOException e) {
throw new FetchingDataFromAdapterException("IO Error while retrieving data from rrd db", e);
}
}
@Override
public String getEncoding() {
return "UTF-8";
}
@Override
public ZoneId getTimeZoneId() {
return ZoneId.systemDefault();
}
@Override
public String getSourceName() {
return "[RRD] " + rrdPaths.get(0).getFileName() + (rrdPaths.size() > 1 ? " + " + (rrdPaths.size() - 1) +
" more RRD file(s)" : "");
}
@Override
public Map<String, String> getParams() {
Map<String, String> params = new HashMap<>();
int i = 0;
for (Path rrdPath : rrdPaths) {
params.put("rrdPaths_" + i++, rrdPath.toString());
}
return params;
}
@Override
public void loadParams(Map<String, String> params) throws DataAdapterException {
this.rrdPaths = params.entrySet().stream()
.filter(entry -> entry.getKey().startsWith("rrdPaths_"))
.map(e -> Paths.get(e.getValue()))
.collect(Collectors.toList());
}
@Override
public void close() {
closeRrdDb();
cleanTempFiles();
super.close();
}
private RrdDb openRrdDb(Path rrdPath) throws IOException {
var factory = RrdBackendFactory.getFactory(prefs.rrd4jBackend.get().toString());
logger.debug(()-> "Opening rrd file using backend factory= " + factory.getName());
if ("text/xml".equalsIgnoreCase(Files.probeContentType(rrdPath))) {
logger.debug(() -> "Attempting to import as an rrd XML dump");
Path temp = Files.createTempFile(UserPreferences.getInstance().temporaryFilesRoot.get(), "binjr_", "_imported.rrd");
tempPathToCollect.add(temp);
return RrdDb.getBuilder()
.setBackendFactory(factory)
.setPath(temp.toUri())
.setReadOnly(true)
.setExternalPath(RrdDb.PREFIX_XML + rrdPath.toString())
.build();
}
try {
return RrdDb.getBuilder()
.setBackendFactory(factory)
.setPath(rrdPath.toUri())
.setReadOnly(true)
.build();
} catch (InvalidRrdException e) {
// Possibly a rrd db created with RrdTool.
// Try to convert and import.
logger.debug(() -> "Failed to open " + rrdPath + " as an Rrd4j db: attempting to import as an rrdTool db");
Path temp = Files.createTempFile(UserPreferences.getInstance().temporaryFilesRoot.get(), "binjr_", "_imported.rrd");
tempPathToCollect.add(temp);
return RrdDb.getBuilder()
.setBackendFactory(factory)
.setPath(temp.toUri())
.setReadOnly(true)
.setExternalPath(RrdDb.PREFIX_RRDTool + rrdPath.toString())
.build();
}
}
private void closeRrdDb() {
rrdDbMap.forEach((s, rrdDb) -> {
logger.debug(() -> "Closing RRD db " + s);
try {
rrdDb.close();
} catch (IOException e) {
logger.error("Error attempting to close RRD db " + s, e);
}
});
rrdDbMap.clear();
}
private void cleanTempFiles() {
//Cleaning up temp files used to import rrdtool files
tempPathToCollect.forEach(p -> {
logger.debug(() -> "Deleting temp file " + p);
try {
Files.delete(p);
} catch (IOException e) {
logger.error("Failed to delete temp file", e);
}
});
tempPathToCollect.clear();
}
}
@@ -0,0 +1,154 @@
/*
* Copyright 2018-2023 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.rrd4j.adapters;
import eu.binjr.common.javafx.controls.NodeUtils;
import eu.binjr.common.logging.Logger;
import eu.binjr.common.preferences.MostRecentlyUsedList;
import eu.binjr.core.data.adapters.DataAdapter;
import eu.binjr.core.data.exceptions.CannotInitializeDataAdapterException;
import eu.binjr.core.data.exceptions.DataAdapterException;
import eu.binjr.core.dialogs.Dialogs;
import eu.binjr.core.preferences.AppEnvironment;
import eu.binjr.core.preferences.UserHistory;
import javafx.application.Platform;
import javafx.event.ActionEvent;
import javafx.geometry.Pos;
import javafx.scene.Node;
import javafx.scene.control.*;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Region;
import javafx.stage.FileChooser;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;
/**
* A dialog box that returns a {@link Rrd4jFileAdapterDialog} built according to user inputs.
*
* @author Frederic Thevenet
*/
public class Rrd4jFileAdapterDialog extends Dialog<Collection<DataAdapter>> {
private static final Logger logger = Logger.create(Rrd4jFileAdapterDialog.class);
private static final String BINJR_SOURCES = "binjr/sources";
private Collection<DataAdapter> result = null;
private final TextField pathsField;
private final MostRecentlyUsedList<Path> mostRecentRrdFiles =
UserHistory.getInstance().pathMostRecentlyUsedList("mostRecentRrdFiles", 20, false);
/**
* Initializes a new instance of the {@link Rrd4jFileAdapterDialog} class.
*
* @param owner the owner window for the dialog
*/
public Rrd4jFileAdapterDialog(Node owner) {
if (owner != null) {
this.initOwner(NodeUtils.getStage(owner));
}
this.setTitle("Source");
Button browseButton = new Button("Browse");
pathsField = new TextField();
HBox pathHBox = new HBox();
pathHBox.setSpacing(10);
pathHBox.setAlignment(Pos.CENTER);
pathHBox.getChildren().addAll(pathsField, browseButton);
browseButton.setPrefWidth(-1);
pathsField.setPrefWidth(400);
DialogPane dialogPane = new DialogPane();
dialogPane.setHeaderText("Add RRD file(s)");
dialogPane.getButtonTypes().addAll(ButtonType.OK, ButtonType.CANCEL);
dialogPane.setGraphic(new Region());
dialogPane.getGraphic().getStyleClass().addAll("source-icon", "dialog-icon");
dialogPane.setContent(pathHBox);
this.setDialogPane(dialogPane);
browseButton.setOnAction(event -> {
File selectedFile = displayFileChooser((Node) event.getSource());
if (selectedFile != null) {
pathsField.setText(selectedFile.getPath());
}
});
Button okButton = (Button) dialogPane.lookupButton(ButtonType.OK);
Platform.runLater(pathsField::requestFocus);
okButton.addEventFilter(ActionEvent.ACTION, ae -> {
try {
result = List.of(getDataAdapter());
} catch (CannotInitializeDataAdapterException e) {
Dialogs.notifyError("Error initializing adapter to source", e, Pos.CENTER, pathsField);
ae.consume();
} catch (DataAdapterException e) {
Dialogs.notifyError("Error with the adapter to source", e, Pos.CENTER, pathsField);
ae.consume();
} catch (Throwable e) {
Dialogs.notifyError("Unexpected error while retrieving data adapter", e, Pos.CENTER, pathsField);
ae.consume();
}
});
this.setResultConverter(dialogButton -> {
ButtonBar.ButtonData data = dialogButton == null ? null : dialogButton.getButtonData();
if (data == ButtonBar.ButtonData.OK_DONE) {
return result;
}
return null;
}
);
// Workaround JDK-8179073 (ref: https://bugs.openjdk.java.net/browse/JDK-8179073)
this.setResizable(AppEnvironment.getInstance().isResizableDialogs());
}
private File displayFileChooser(Node owner) {
try {
FileChooser fileChooser = new FileChooser();
fileChooser.setTitle("Open Rrd4j File(s)");
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("RRD binary files", "*.rrd"));
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("RRD XML dumps", "*.xml"));
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("All files", "*.*", "*"));
Dialogs.getInitialDir(mostRecentRrdFiles).ifPresent(fileChooser::setInitialDirectory);
List<File> rrdFiles = fileChooser.showOpenMultipleDialog(NodeUtils.getStage(owner));
if (rrdFiles != null) {
pathsField.setText(rrdFiles.stream().map(File::getPath).collect(Collectors.joining(";")));
}
} catch (Exception e) {
Dialogs.notifyException("Error while displaying file chooser: " + e.getMessage(), e, owner);
}
return null;
}
/**
* Returns an instance of {@link Rrd4jFileAdapter}
*
* @return an instance of {@link Rrd4jFileAdapter}
* @throws DataAdapterException if the provided parameters are invalid
*/
private DataAdapter<Double> getDataAdapter() throws DataAdapterException {
if (pathsField.getText().isBlank()){
throw new CannotInitializeDataAdapterException("Path cannot be blank");
}
List<Path> rrdFiles = Arrays.stream(pathsField.getText().split(";")).map(Paths::get).collect(Collectors.toList());
rrdFiles.forEach(mostRecentRrdFiles::push);
return new Rrd4jFileAdapter(rrdFiles);
}
}
@@ -0,0 +1,46 @@
/*
* Copyright 2020 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.rrd4j.adapters;
import eu.binjr.common.preferences.ObservablePreference;
import eu.binjr.core.data.adapters.DataAdapter;
import eu.binjr.core.data.adapters.DataAdapterPreferences;
/**
* Defines the preferences associated with the RRD4J adapter.
*
* @author Frederic Thevenet
*/
public class Rrd4jFileAdapterPreferences extends DataAdapterPreferences {
/**
* The backend to be used by RRD4J to mount rrd files.
*/
public final ObservablePreference<Rrd4jBackendType> rrd4jBackend =
enumPreference(Rrd4jBackendType.class, "rrd4jBackend", Rrd4jBackendType.NIO);
/**
* Initialize a new instance of the {@link Rrd4jFileAdapterPreferences} class associated to
* a {@link DataAdapter} instance.
*
* @param dataAdapterClass the associated {@link DataAdapter}
*/
public Rrd4jFileAdapterPreferences(Class<? extends DataAdapter<?>> dataAdapterClass) {
super(dataAdapterClass);
}
}
@@ -0,0 +1,54 @@
/*
* Copyright 2018-2023 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.rrd4j.adapters;
import eu.binjr.core.data.adapters.AdapterMetadata;
import eu.binjr.core.data.adapters.BaseDataAdapterInfo;
import eu.binjr.core.data.adapters.SourceLocality;
import eu.binjr.core.data.adapters.VisualizationType;
import eu.binjr.core.data.exceptions.CannotInitializeDataAdapterException;
import eu.binjr.core.preferences.AppEnvironment;
/**
* Defines the metadata associated with the Rrd4jFileDataAdapter
*
* @author Frederic Thevenet
*/
@AdapterMetadata(
name = "RRD",
description = "RRD Data Adapter",
copyright = AppEnvironment.COPYRIGHT_NOTICE,
license = AppEnvironment.LICENSE,
siteUrl = AppEnvironment.HTTP_WWW_BINJR_EU,
adapterClass = Rrd4jFileAdapter.class,
dialogClass = Rrd4jFileAdapterDialog.class,
preferencesClass = Rrd4jFileAdapterPreferences.class,
sourceLocality = SourceLocality.LOCAL,
apiLevel = AppEnvironment.PLUGIN_API_LEVEL,
visualizationType = VisualizationType.CHARTS
)
public class Rrd4jFileDataAdapterInfo extends BaseDataAdapterInfo {
/**
* Initialises a new instance of the {@link Rrd4jFileDataAdapterInfo} class.
*
* @throws CannotInitializeDataAdapterException if an error occurs initializing the adapter.
*/
public Rrd4jFileDataAdapterInfo() throws CannotInitializeDataAdapterException {
super(Rrd4jFileDataAdapterInfo.class);
}
}
@@ -0,0 +1,18 @@
#
# Copyright 2018 Frederic Thevenet
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# RRD file Data adapter service implementation
eu.binjr.sources.rrd4j.adapters.Rrd4jFileDataAdapterInfo