admin/AndroidChart
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

的

Browse Source
This commit is contained in:
coco
2026-07-03 15:56:07 +08:00
commit caef23209c
5767 changed files with 1004268 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
binjr/binjr-adapter-logs/src/main/java/eu/binjr/sources/logs/adapters/LogsAdapterPreferences.java
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright 2020 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.logs.adapters;
import com.google.gson.Gson;
import eu.binjr.common.preferences.ObservablePreference;
import eu.binjr.core.data.adapters.DataAdapter;
import eu.binjr.core.data.adapters.DataAdapterPreferences;
import eu.binjr.core.data.indexes.parser.profile.BuiltInParsingProfile;
import eu.binjr.core.data.indexes.parser.profile.ParsingProfile;
import java.nio.charset.StandardCharsets;
/**
* Defines the preferences associated with the Log files adapter.
*
* @author Frederic Thevenet
*/
public class LogsAdapterPreferences extends DataAdapterPreferences {
private static final Gson gson = new Gson();
/**
* The default text panel font size preference.
*/
public ObservablePreference<Number> defaultTextViewFontSize = integerPreference("defaultTextViewFontSize", 10);
/**
* The filters used when scanning folders in the source filesystem.
*/
public ObservablePreference<String[]> folderFilters = objectPreference(String[].class,
"folderFilters",
new String[]{"*"},
gson::toJson,
s -> gson.fromJson(s, String[].class));
/**
* The filters used to prune file extensions to scan in the source filesystem.
*/
public ObservablePreference<String[]> fileExtensionFilters = objectPreference(String[].class,
"extensionFilters",
new String[]{"*.log", "*.txt"},
gson::toJson,
s -> gson.fromJson(s, String[].class));
/**
* The most recently used {@link ParsingProfile}
*/
public ObservablePreference<String> mostRecentlyUsedParsingProfile =
stringPreference("mostRecentlyUsedParsingProfile", BuiltInParsingProfile.ISO.getProfileId());
/**
* The most recently used encoding
*/
public ObservablePreference<String> mruEncoding =
stringPreference("mruEncoding", StandardCharsets.UTF_8.name());
/**
* Initialize a new instance of the {@link LogsAdapterPreferences} class associated to
* a {@link DataAdapter} instance.
*
* @param dataAdapterClass the associated {@link DataAdapter}
*/
public LogsAdapterPreferences(Class<? extends DataAdapter<?>> dataAdapterClass) {
super(dataAdapterClass);
}
}
binjr/binjr-adapter-logs/src/main/java/eu/binjr/sources/logs/adapters/LogsDataAdapter.java
+473
View File
@@ -0,0 +1,473 @@
/*
* Copyright 2020-2023 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.logs.adapters;
import com.google.gson.Gson;
import eu.binjr.common.function.CheckedFunction;
import eu.binjr.common.function.CheckedLambdas;
import eu.binjr.common.io.FileSystemBrowser;
import eu.binjr.common.io.IOUtils;
import eu.binjr.common.javafx.controls.TimeRange;
import eu.binjr.common.javafx.controls.TreeViewUtils;
import eu.binjr.common.logging.Logger;
import eu.binjr.common.logging.Profiler;
import eu.binjr.common.preferences.MostRecentlyUsedList;
import eu.binjr.common.text.BinaryPrefixFormatter;
import eu.binjr.core.data.adapters.*;
import eu.binjr.core.data.exceptions.CannotInitializeDataAdapterException;
import eu.binjr.core.data.exceptions.DataAdapterException;
import eu.binjr.core.data.indexes.*;
import eu.binjr.core.data.indexes.parser.EventFormat;
import eu.binjr.core.data.indexes.parser.LogEventFormat;
import eu.binjr.core.data.indexes.parser.profile.CustomParsingProfile;
import eu.binjr.core.data.indexes.parser.profile.ParsingProfile;
import eu.binjr.core.data.timeseries.TimeSeriesProcessor;
import eu.binjr.core.data.workspace.LogFileSeriesInfo;
import eu.binjr.core.data.workspace.TimeSeriesInfo;
import eu.binjr.core.dialogs.Dialogs;
import eu.binjr.core.preferences.UserHistory;
import javafx.beans.property.*;
import javafx.beans.value.ChangeListener;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.FacetField;
import org.eclipse.fx.ui.controls.tree.FilterableTreeItem;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.ZoneId;
import java.util.*;
import java.util.stream.Collectors;
import static eu.binjr.core.data.indexes.parser.capture.CaptureGroup.SEVERITY;
/**
* A {@link DataAdapter} implementation to retrieve data from a text file.
*
* @author Frederic Thevenet
*/
public class LogsDataAdapter extends BaseDataAdapter<SearchHit> implements ProgressAdapter<SearchHit> {
private static final Logger logger = Logger.create(LogsDataAdapter.class);
private static final Gson gson = new Gson();
private static final String DEFAULT_PREFIX = "[Logs]";
private static final String ZONE_ID_PARAM_NAME = "zoneId";
private static final String ROOT_PATH_PARAM_NAME = "rootPath";
private static final String FOLDER_FILTERS_PARAM_NAME = "folderFilters";
private static final String EXTENSIONS_FILTERS_PARAM_NAME = "fileExtensionsFilters";
private static final String PARSING_PROFILE_PARAM_NAME = "parsingProfile";
private static final String LOG_FILE_ENCODING = "LOG_FILE_ENCODING";
private static final Property<IndexingStatus> INDEXING_OK = new ReadOnlyObjectWrapper(new SimpleObjectProperty<>(IndexingStatus.OK));
private final String sourceNamePrefix;
private final Map<String, IndexingStatus> indexedFiles = new HashMap<>();
private final BinaryPrefixFormatter binaryPrefixFormatter = new BinaryPrefixFormatter("###,###.## ");
private final MostRecentlyUsedList<String> defaultParsingProfiles =
UserHistory.getInstance().stringMostRecentlyUsedList("defaultParsingProfiles", 100);
private final MostRecentlyUsedList<String> userParsingProfiles =
UserHistory.getInstance().stringMostRecentlyUsedList("userParsingProfiles", 100);
private final Charset encoding;
private Path rootPath;
private Indexable index;
private FileSystemBrowser fileBrowser;
private String[] folderFilters;
private String[] fileExtensionsFilters;
private ParsingProfile parsingProfile;
private EventFormat<InputStream> defaultEventFormat;
private ZoneId zoneId;
/**
* Initializes a new instance of the {@link LogsDataAdapter} class.
*
* @throws DataAdapterException if an error occurs while initializing the adapter.
*/
public LogsDataAdapter() throws DataAdapterException {
super();
zoneId = ZoneId.systemDefault();
encoding = StandardCharsets.UTF_8;
sourceNamePrefix = DEFAULT_PREFIX;
}
/**
* Initializes a new instance of the {@link LogsDataAdapter} class from the provided {@link Path}
*
* @param rootPath the {@link Path} from which to load content.
* @param folderFilters a list of names of folders to inspect for content.
* @param fileExtensionsFilters a list of file extensions to inspect for content.
* @param profile the parsing profile to use.
* @throws DataAdapterException if an error occurs initializing the adapter.
*/
public LogsDataAdapter(Path rootPath,
ZoneId zoneId,
String[] folderFilters,
String[] fileExtensionsFilters,
ParsingProfile profile) throws DataAdapterException {
this(DEFAULT_PREFIX, rootPath, zoneId, StandardCharsets.UTF_8, folderFilters, fileExtensionsFilters, profile);
}
/**
* Initializes a new instance of the {@link LogsDataAdapter} class from the provided {@link Path}
*
* @param sourcePrefix the name to prepend the source with.
* @param rootPath the {@link Path} from which to load content.
* @param folderFilters a list of names of folders to inspect for content.
* @param fileExtensionsFilters a list of file extensions to inspect for content.
* @param profile the parsing profile to use.
* @throws DataAdapterException if an error occurs initializing the adapter.
*/
public LogsDataAdapter(String sourcePrefix,
Path rootPath,
ZoneId zoneId,
String[] folderFilters,
String[] fileExtensionsFilters,
ParsingProfile profile) throws DataAdapterException {
this(sourcePrefix, rootPath, zoneId, StandardCharsets.UTF_8, folderFilters, fileExtensionsFilters, profile);
}
/**
* Initializes a new instance of the {@link LogsDataAdapter} class from the provided {@link Path}
*
* @param rootPath the {@link Path} from which to load content.
* @param folderFilters a list of names of folders to inspect for content.
* @param fileExtensionsFilters a list of file extensions to inspect for content.
* @param profile the parsing profile to use.
* @throws DataAdapterException if an error occurs initializing the adapter.
*/
public LogsDataAdapter(Path rootPath,
ZoneId zoneId,
Charset encoding,
String[] folderFilters,
String[] fileExtensionsFilters,
ParsingProfile profile) throws DataAdapterException {
this(DEFAULT_PREFIX, rootPath, zoneId, encoding, folderFilters, fileExtensionsFilters, profile);
}
/**
* Initializes a new instance of the {@link LogsDataAdapter} class from the provided {@link Path}
*
* @param sourcePrefix the name to prepend the source with.
* @param rootPath the {@link Path} from which to load content.
* @param folderFilters a list of names of folders to inspect for content.
* @param fileExtensionsFilters a list of file extensions to inspect for content.
* @param profile the parsing profile to use.
* @throws DataAdapterException if an error occurs initializing the adapter.
*/
public LogsDataAdapter(String sourcePrefix,
Path rootPath,
ZoneId zoneId,
Charset encoding,
String[] folderFilters,
String[] fileExtensionsFilters,
ParsingProfile profile) throws DataAdapterException {
super();
this.sourceNamePrefix = sourcePrefix;
this.rootPath = rootPath;
this.encoding = encoding;
Map<String, String> params = new HashMap<>();
initParams(rootPath, zoneId, folderFilters, fileExtensionsFilters, profile);
}
@Override
public Map<String, String> getParams() {
Map<String, String> params = new HashMap<>();
params.put(LOG_FILE_ENCODING, getEncoding());
params.put(ROOT_PATH_PARAM_NAME, rootPath.toString());
params.put(ZONE_ID_PARAM_NAME, zoneId.toString());
params.put(FOLDER_FILTERS_PARAM_NAME, gson.toJson(folderFilters));
params.put(EXTENSIONS_FILTERS_PARAM_NAME, gson.toJson(fileExtensionsFilters));
params.put(PARSING_PROFILE_PARAM_NAME, gson.toJson(CustomParsingProfile.of(parsingProfile)));
return params;
}
@Override
public void loadParams(Map<String, String> params) throws DataAdapterException {
if (logger.isDebugEnabled()) {
logger.debug(() -> "LogsDataAdapter params:");
params.forEach((s, s2) -> logger.debug(() -> "key=" + s + ", value=" + s2));
}
initParams(Paths.get(validateParameterNullity(params, ROOT_PATH_PARAM_NAME)),
validateParameter(params, ZONE_ID_PARAM_NAME,
s -> {
if (s == null) {
logger.warn("Parameter " + ZONE_ID_PARAM_NAME + " is missing in adapter " + getSourceName());
return ZoneId.systemDefault();
}
return ZoneId.of(s);
}),
gson.fromJson(validateParameterNullity(params, FOLDER_FILTERS_PARAM_NAME), String[].class),
gson.fromJson(validateParameterNullity(params, EXTENSIONS_FILTERS_PARAM_NAME), String[].class),
gson.fromJson(validateParameterNullity(params, PARSING_PROFILE_PARAM_NAME), CustomParsingProfile.class));
}
private void initParams(Path rootPath,
ZoneId zoneId,
String[] folderFilters,
String[] fileExtensionsFilters,
ParsingProfile parsingProfile) throws DataAdapterException {
this.rootPath = rootPath;
this.zoneId = zoneId;
this.folderFilters = folderFilters;
this.fileExtensionsFilters = fileExtensionsFilters;
this.parsingProfile = parsingProfile;
this.defaultEventFormat = new LogEventFormat(parsingProfile, getTimeZoneId(), encoding);
}
@Override
public void onStart() throws DataAdapterException {
super.onStart();
try {
this.fileBrowser = FileSystemBrowser.of(rootPath);
this.index = Indexes.LOG_FILES.acquire();
} catch (IOException e) {
throw new CannotInitializeDataAdapterException("An error occurred during the data adapter initialization", e);
}
}
@Override
public FilterableTreeItem<SourceBinding> getBindingTree() throws DataAdapterException {
FilterableTreeItem<SourceBinding> configNode = new FilterableTreeItem<>(
new LogFilesBinding.Builder()
.withLabel(getSourceName())
.withAdapter(this)
.build());
attachNodes(configNode);
return configNode;
}
private void attachNodes(FilterableTreeItem<SourceBinding> root) throws DataAdapterException {
try (var p = Profiler.start("Building log files binding tree", logger::perf)) {
Map<Path, FilterableTreeItem<SourceBinding>> nodeDict = new HashMap<>();
nodeDict.put(fileBrowser.toInternalPath("/"), root);
for (var fsEntry : fileBrowser.listEntries(folderFilters, fileExtensionsFilters)) {
String fileName = fsEntry.getPath().getFileName().toString();
var attachTo = root;
if (fsEntry.getPath().getParent() != null) {
attachTo = nodeDict.get(fsEntry.getPath().getParent());
if (attachTo == null) {
attachTo = makeBranchNode(nodeDict, fsEntry.getPath().getParent(), root);
}
}
FilterableTreeItem<SourceBinding> filenode = new FilterableTreeItem<>(
new LogFilesBinding.Builder()
.withLabel(fileName + " (" + binaryPrefixFormatter.format(fsEntry.getSize()) + "B)")
.withPath(getId() + "/" + fsEntry.getPath().toString())
.withParent(attachTo.getValue())
.withParsingProfile(parsingProfile)
.withAdapter(this)
.build());
attachTo.getInternalChildren().add(filenode);
}
TreeViewUtils.sortFromBranch(root);
} catch (Exception e) {
Dialogs.notifyException("Error while enumerating files: " + e.getMessage(), e);
}
}
private FilterableTreeItem<SourceBinding> makeBranchNode(Map<Path, FilterableTreeItem<SourceBinding>> nodeDict,
Path path,
FilterableTreeItem<SourceBinding> root) {
var parent = root;
var rootPath = path.isAbsolute() ? path.getRoot() : path.getName(0);
for (int i = 0; i < path.getNameCount(); i++) {
Path current = rootPath.resolve(path.getName(i));
FilterableTreeItem<SourceBinding> filenode = nodeDict.get(current);
if (filenode == null) {
filenode = new FilterableTreeItem<>(
new LogFilesBinding.Builder()
.withLabel(current.getFileName().toString())
.withPath(getId() + "/" + path.toString())
.withParent(parent.getValue())
.withParsingProfile(parsingProfile)
.withAdapter(this)
.build());
nodeDict.put(current, filenode);
parent.getInternalChildren().add(filenode);
}
parent = filenode;
rootPath = current;
}
return parent;
}
@Override
public TimeRange getInitialTimeRange(String path, List<TimeSeriesInfo<SearchHit>> seriesInfo) throws DataAdapterException {
try {
return index.getTimeRangeBoundaries(seriesInfo.stream().map(this::getPathFacetValue).toList(), getTimeZoneId());
} catch (IOException e) {
throw new DataAdapterException("Error retrieving initial time range", e);
}
}
@Deprecated
@Override
public Map<TimeSeriesInfo<SearchHit>, TimeSeriesProcessor<SearchHit>> fetchData(String path,
Instant begin,
Instant end,
List<TimeSeriesInfo<SearchHit>> seriesInfo,
boolean bypassCache) throws DataAdapterException {
return loadSeries(path, seriesInfo, bypassCache ? ReloadPolicy.ALL : ReloadPolicy.UNLOADED, null, INDEXING_OK);
}
@Override
public Map<TimeSeriesInfo<SearchHit>, TimeSeriesProcessor<SearchHit>> loadSeries(String path,
List<TimeSeriesInfo<SearchHit>> seriesInfo,
ReloadPolicy reloadPolicy,
DoubleProperty progress,
Property<IndexingStatus> indexingStatus) throws DataAdapterException {
Map<TimeSeriesInfo<SearchHit>, TimeSeriesProcessor<SearchHit>> data = new HashMap<>();
try {
ensureIndexed(seriesInfo.stream()
.filter(s -> s instanceof LogFileSeriesInfo)
.map(s -> (LogFileSeriesInfo) s)
.toList(),
progress,
reloadPolicy,
indexingStatus);
} catch (Exception e) {
throw new DataAdapterException("Error fetching logs from " + path, e);
}
return data;
}
private synchronized void ensureIndexed(List<LogFileSeriesInfo> seriesInfo,
DoubleProperty progress,
ReloadPolicy reloadPolicy,
Property<IndexingStatus> indexingStatus) throws IOException {
final var toDo = seriesInfo.stream()
.filter(p -> switch (reloadPolicy) {
case ALL -> true;
case UNLOADED -> !indexedFiles.containsKey(getPathFacetValue(p));
case INCOMPLETE ->
indexedFiles.getOrDefault(getPathFacetValue(p), IndexingStatus.CANCELED) == IndexingStatus.CANCELED;
})
.toList();
if (toDo.size() > 0) {
final long totalSizeInBytes = toDo.stream()
.map(CheckedLambdas.wrap((CheckedFunction<LogFileSeriesInfo, Long, IOException>)
e -> fileBrowser.getEntry(e.getBinding().getPath().replace(getId() + "/", "")).getSize()))
.reduce(Long::sum).orElse(0L);
final ChangeListener<Number> progressListener = (observable, oldValue, newValue) -> {
if (newValue != null && totalSizeInBytes > 0) {
var oldProgress = (oldValue.longValue() * 100 / totalSizeInBytes) / 100.0;
var newProgress = (newValue.longValue() * 100 / totalSizeInBytes) / 100.0;
if (progress != null && oldProgress != newProgress) {
Dialogs.runOnFXThread(() -> progress.setValue(newProgress));
}
}
};
final LongProperty charRead = new SimpleLongProperty(0);
charRead.addListener(progressListener);
try {
for (int i = 0; i < toDo.size(); i++) {
var tsInfo = toDo.get(i);
String path = tsInfo.getBinding().getPath();
var key = getPathFacetValue(tsInfo);
index.add(key,
fileBrowser.getData(path.replace(getId() + "/", "")),
(i == toDo.size() - 1), // commit if last file
getEventFormat(tsInfo),
(doc, event) -> {
// add all other sections as prefixed search fields
event.getTextFields().entrySet().stream()
.filter(e -> !e.getKey().equals(SEVERITY))
.forEach(e -> doc.add(new TextField(e.getKey(), e.getValue(), Field.Store.NO)));
// Add severity
String severity = event.getTextField(SEVERITY) == null ? "unknown" : event.getTextField(SEVERITY).toLowerCase();
doc.add(new FacetField(SEVERITY, severity));
doc.add(new StoredField(SEVERITY, severity));
return doc;
},
charRead,
indexingStatus);
indexedFiles.put(key, indexingStatus.getValue());
}
} finally {
// remove listener
charRead.removeListener(progressListener);
if (progress != null) {
Dialogs.runOnFXThread(() -> progress.setValue(-1));
}
// reset cancellation request
indexingStatus.setValue(IndexingStatus.OK);
}
}
// Update loading status for series
for (var series : seriesInfo) {
series.setIndexingStatus(indexedFiles.get(getPathFacetValue(series)));
}
}
private String readTextFile(String path) throws IOException {
try (Profiler ignored = Profiler.start("Extracting text from file " + path, logger::perf)) {
try (var reader = new BufferedReader(new InputStreamReader(fileBrowser.getData(path), StandardCharsets.UTF_8))) {
return reader.lines().collect(Collectors.joining("\n"));
}
}
}
private String getPathFacetValue(TimeSeriesInfo<?> p) {
if (p instanceof LogFileSeriesInfo lfsi && lfsi.getParsingProfile() != null) {
return lfsi.getPathFacetValue();
}
return LogFileSeriesInfo.makePathFacetValue(parsingProfile, p);
}
private EventFormat<InputStream> getEventFormat(TimeSeriesInfo<?> p) {
if (p instanceof LogFileSeriesInfo lfsi && lfsi.getParsingProfile() != null) {
return new LogEventFormat(lfsi.getParsingProfile(), getTimeZoneId(), encoding);
}
return defaultEventFormat;
}
@Override
public String getEncoding() {
return encoding.name();
}
@Override
public ZoneId getTimeZoneId() {
return zoneId;
}
@Override
public String getSourceName() {
return String.format("%s %s", sourceNamePrefix, rootPath != null ? rootPath.getFileName() : "???");
}
@Override
public void close() {
try {
Indexes.LOG_FILES.release();
} catch (Exception e) {
logger.error("An error occurred while releasing index " + Indexes.LOG_FILES.name() + ": " + e.getMessage());
logger.debug("Stack Trace:", e);
}
IOUtils.close(fileBrowser);
super.close();
}
}
binjr/binjr-adapter-logs/src/main/java/eu/binjr/sources/logs/adapters/LogsDataAdapterDialog.java
+221
View File
@@ -0,0 +1,221 @@
/*
* Copyright 2020-2022 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.logs.adapters;
import eu.binjr.common.javafx.controls.LabelWithInlineHelp;
import eu.binjr.common.javafx.controls.NodeUtils;
import eu.binjr.common.logging.Logger;
import eu.binjr.core.data.adapters.DataAdapter;
import eu.binjr.core.data.adapters.DataAdapterFactory;
import eu.binjr.core.data.exceptions.CannotInitializeDataAdapterException;
import eu.binjr.core.data.exceptions.DataAdapterException;
import eu.binjr.core.data.exceptions.NoAdapterFoundException;
import eu.binjr.core.data.indexes.parser.profile.BuiltInParsingProfile;
import eu.binjr.core.data.indexes.parser.profile.ParsingProfile;
import eu.binjr.core.dialogs.DataAdapterDialog;
import eu.binjr.core.dialogs.Dialogs;
import eu.binjr.core.dialogs.LogParsingProfileDialog;
import eu.binjr.core.preferences.UserPreferences;
import javafx.geometry.*;
import javafx.scene.Node;
import javafx.scene.control.*;
import javafx.scene.layout.GridPane;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Priority;
import javafx.stage.DirectoryChooser;
import javafx.stage.FileChooser;
import org.controlsfx.control.textfield.TextFields;
import java.io.File;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
/**
* An implementation of the {@link DataAdapterDialog} class that presents a dialog box to retrieve the parameters specific {@link LogsDataAdapterDialog}
*
* @author Frederic Thevenet
*/
public class LogsDataAdapterDialog extends DataAdapterDialog<Path> {
private static final Logger logger = Logger.create(LogsDataAdapterDialog.class);
private final TextField extensionFiltersTextField;
private final TextField encodingField;
private final LogsAdapterPreferences prefs;
private final ChoiceBox<ParsingProfile> parsingChoiceBox = new ChoiceBox<>();
private void updateProfileList(ParsingProfile[] newValue) {
parsingChoiceBox.getItems().clear();
parsingChoiceBox.getItems().setAll(BuiltInParsingProfile.editableProfiles());
parsingChoiceBox.getItems().addAll(newValue);
parsingChoiceBox.getSelectionModel().select(parsingChoiceBox.getItems().stream()
.filter(p -> Objects.equals(p.getProfileId(), prefs.mostRecentlyUsedParsingProfile.get()))
.findAny().orElse(BuiltInParsingProfile.ALL));
}
/**
* Initializes a new instance of the {@link LogsDataAdapterDialog} class.
*
* @param owner the owner window for the dialog
* @throws NoAdapterFoundException if an error occurs initializing the dialog.
*/
public LogsDataAdapterDialog(Node owner) throws NoAdapterFoundException {
super(owner, Mode.PATH, "mostRecentLogsArchives", true);
this.prefs = (LogsAdapterPreferences) DataAdapterFactory.getInstance().getAdapterPreferences(LogsDataAdapter.class.getName());
setDialogHeaderText("Add a Log File, Zip Archive or Folder");
this.setUriLabelInlineHelp("""
A file system path to get log files from.
It can either be:
- The path to a single text-based log file, which will then be the sole item available in the source tree.
- The path to a folder, in which case all files in the folder and sub-folders that match the extension filter below will be available the the source tree.
- The path to a Zip archive, in which case all files in the archive that match the extension filter below will be available the the source tree.
""");
this.setTimezoneLabelInlineHelp("""
The default timezone for timestamps in the source files.
""");
extensionFiltersTextField = new TextField(String.join(", ", prefs.fileExtensionFilters.get()));
var extensionlabel = new LabelWithInlineHelp();
extensionlabel.setText("Extensions");
extensionlabel.setInlineHelp("""
The set of extensions used to filter which files should be available in the source tree when pointing to a folder or a zip archive.
You can specify any number of glob patterns separated by spaces, commas or semi-colons.
""");
extensionlabel.setAlignment(Pos.CENTER_RIGHT);
GridPane.setConstraints(extensionlabel, 0, 2, 1, 1, HPos.LEFT, VPos.CENTER, Priority.ALWAYS, Priority.ALWAYS, new Insets(4, 0, 4, 0));
GridPane.setConstraints(extensionFiltersTextField, 1, 2, 1, 1, HPos.LEFT, VPos.CENTER, Priority.ALWAYS, Priority.ALWAYS, new Insets(4, 0, 4, 0));
this.encodingField = new TextField(prefs.mruEncoding.get());
TextFields.bindAutoCompletion(encodingField, Charset.availableCharsets().keySet());
GridPane.setConstraints(encodingField, 1, 3, 1, 1, HPos.LEFT, VPos.CENTER, Priority.ALWAYS, Priority.ALWAYS, new Insets(4, 0, 4, 0));
var encodingLabel = new LabelWithInlineHelp();
encodingLabel.setText("Encoding");
encodingLabel.setInlineHelp("""
The text encoding of files to extracts log events from.
""");
encodingLabel.setAlignment(Pos.CENTER_RIGHT);
GridPane.setConstraints(encodingLabel, 0, 3, 1, 1, HPos.LEFT, VPos.CENTER, Priority.ALWAYS, Priority.ALWAYS, new Insets(4, 0, 4, 0));
var parsingLabel = new LabelWithInlineHelp();
parsingLabel.setText("Parsing profile");
parsingLabel.setInlineHelp("""
The default parsing profile used to process log files loaded from this source.
It is still possible to select a different profile on of file-by-file basis once the source is loaded.
""");
parsingLabel.setAlignment(Pos.CENTER_RIGHT);
var parsingHBox = new HBox();
parsingHBox.setSpacing(5);
updateProfileList(UserPreferences.getInstance().userLogEventsParsingProfiles.get());
parsingChoiceBox.setMaxWidth(Double.MAX_VALUE);
var editParsingButton = new Button("Edit");
editParsingButton.setOnAction(event -> {
try {
new LogParsingProfileDialog(this.getOwner(), parsingChoiceBox.getValue()).showAndWait().ifPresent(selection -> {
prefs.mostRecentlyUsedParsingProfile.set(selection.getProfileId());
updateProfileList( UserPreferences.getInstance().userLogEventsParsingProfiles.get());
});
} catch (Exception e) {
Dialogs.notifyException("Failed to show parsing profile windows", e, owner);
}
});
parsingHBox.getChildren().addAll(parsingChoiceBox, editParsingButton);
HBox.setHgrow(parsingChoiceBox, Priority.ALWAYS);
GridPane.setConstraints(parsingLabel, 0, 4, 1, 1, HPos.LEFT, VPos.CENTER, Priority.ALWAYS, Priority.ALWAYS, new Insets(4, 0, 4, 0));
GridPane.setConstraints(parsingHBox, 1, 4, 1, 1, HPos.LEFT, VPos.CENTER, Priority.ALWAYS, Priority.ALWAYS, new Insets(4, 0, 4, 0));
getParamsGridPane().getChildren().addAll(extensionlabel, extensionFiltersTextField, parsingLabel, parsingHBox, encodingLabel, encodingField);
}
@Override
protected File displayFileChooser(Node owner) {
try {
ContextMenu sourceMenu = new ContextMenu();
MenuItem fileMenuItem = new MenuItem("Log file");
fileMenuItem.setOnAction(eventHandler -> {
FileChooser fileChooser = new FileChooser();
fileChooser.setTitle("Open Log File");
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("Log file", "*.log", "*.txt", "*.trace", "*.out", "*.err"));
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("All files", "*.*", "*"));
Dialogs.getInitialDir(getMostRecentList()).ifPresent(fileChooser::setInitialDirectory);
File selectedFile = fileChooser.showOpenDialog(NodeUtils.getStage(owner));
if (selectedFile != null) {
setSourceUri(selectedFile.getPath());
}
});
sourceMenu.getItems().add(fileMenuItem);
MenuItem zipMenuItem = new MenuItem("Zip file");
zipMenuItem.setOnAction(eventHandler -> {
FileChooser fileChooser = new FileChooser();
fileChooser.setTitle("Open Zip Archive");
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("Zip archive", "*.zip"));
fileChooser.getExtensionFilters().add(new FileChooser.ExtensionFilter("All files", "*.*", "*"));
Dialogs.getInitialDir(getMostRecentList()).ifPresent(fileChooser::setInitialDirectory);
File selectedFile = fileChooser.showOpenDialog(NodeUtils.getStage(owner));
if (selectedFile != null) {
setSourceUri(selectedFile.getPath());
}
});
sourceMenu.getItems().add(zipMenuItem);
MenuItem folderMenuItem = new MenuItem("Folder");
folderMenuItem.setOnAction(eventHandler -> {
DirectoryChooser dirChooser = new DirectoryChooser();
dirChooser.setTitle("Open Folder");
Dialogs.getInitialDir(getMostRecentList()).ifPresent(dirChooser::setInitialDirectory);
File selectedFile = dirChooser.showDialog(NodeUtils.getStage(owner));
if (selectedFile != null) {
setSourceUri(selectedFile.getPath());
}
});
sourceMenu.getItems().add(folderMenuItem);
sourceMenu.show(owner, Side.RIGHT, 0, 0);
} catch (Exception e) {
Dialogs.notifyException("Error while displaying file chooser: " + e.getMessage(), e, owner);
}
return null;
}
@Override
protected Collection<DataAdapter> getDataAdapters() throws DataAdapterException {
Path path = Paths.get(getSourceUri());
if (!Files.exists(path)) {
throw new CannotInitializeDataAdapterException("Cannot find " + getSourceUri());
}
if (!path.isAbsolute()) {
throw new CannotInitializeDataAdapterException("The provided path is not valid.");
}
getMostRecentList().push(path);
prefs.fileExtensionFilters.set(Arrays.stream(extensionFiltersTextField.getText().split("[,;\" ]+")).filter(e -> !e.isBlank()).toArray(String[]::new));
prefs.mostRecentlyUsedParsingProfile.set(parsingChoiceBox.getValue().getProfileId());
String charsetName = encodingField.getText();
if (!Charset.isSupported(charsetName)) {
throw new CannotInitializeDataAdapterException("Invalid or unsupported encoding: " + charsetName);
}
return List.of(new LogsDataAdapter(path,
ZoneId.of(getSourceTimezone()),
Charset.forName(charsetName),
prefs.folderFilters.get(),
prefs.fileExtensionFilters.get(),
parsingChoiceBox.getValue()));
}
}
binjr/binjr-adapter-logs/src/main/java/eu/binjr/sources/logs/adapters/LogsDataAdapterInfo.java
+57
View File
@@ -0,0 +1,57 @@
/*
* Copyright 2020-2023 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.logs.adapters;
import eu.binjr.common.javafx.controls.ToolButtonBuilder;
import eu.binjr.core.data.adapters.AdapterMetadata;
import eu.binjr.core.data.adapters.BaseDataAdapterInfo;
import eu.binjr.core.data.adapters.SourceLocality;
import eu.binjr.core.data.adapters.VisualizationType;
import eu.binjr.core.data.exceptions.CannotInitializeDataAdapterException;
import eu.binjr.core.preferences.AppEnvironment;
/**
* Defines the metadata associated with LogsDataAdapterInfo.
*
* @author Frederic Thevenet
*/
@AdapterMetadata(
name = "Logs",
description = "Log Files Data Adapter",
copyright = AppEnvironment.COPYRIGHT_NOTICE,
license = AppEnvironment.LICENSE,
siteUrl = AppEnvironment.HTTP_WWW_BINJR_EU,
adapterClass = LogsDataAdapter.class,
dialogClass = LogsDataAdapterDialog.class,
preferencesClass = LogsAdapterPreferences.class,
sourceLocality = SourceLocality.LOCAL,
apiLevel = AppEnvironment.PLUGIN_API_LEVEL,
visualizationType = VisualizationType.EVENTS
)
public class LogsDataAdapterInfo extends BaseDataAdapterInfo {
/**
* Initialises a new instance of the {@link LogsDataAdapterInfo} class.
*
* @throws CannotInitializeDataAdapterException if the adapter's initialization failed
*/
public LogsDataAdapterInfo() throws CannotInitializeDataAdapterException {
super(LogsDataAdapterInfo.class);
}
}
binjr/binjr-adapter-logs/src/main/resources/META-INF/services/eu.binjr.core.data.adapters.DataAdapterInfo
+18
View File
@@ -0,0 +1,18 @@
#
# Copyright 2020 Frederic Thevenet
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Text Data adapter service implementation
eu.binjr.sources.logs.adapters.LogsDataAdapterInfo
binjr/binjr-adapter-logs/src/test/java/eu/binjr/sources/logs/data/parsers/ParserTest.java
+134
View File
@@ -0,0 +1,134 @@
/*
* Copyright 2022 Frederic Thevenet
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.binjr.sources.logs.data.parsers;
import eu.binjr.core.data.indexes.parser.LogEventFormat;
import eu.binjr.core.data.indexes.parser.profile.BuiltInParsingProfile;
import eu.binjr.core.data.indexes.parser.profile.ParsingProfile;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.time.ZoneId;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class ParserTest {
@Test
public void parseBinjrLogsFull() {
assertTrue(parsingTest(BuiltInParsingProfile.ISO, "[2020/07/21-02:56:42.460] [info] Hello World!"));
assertTrue(parsingTest(BuiltInParsingProfile.ISO, "[2020-11-13 19:59:22.627] [INFO ] Hello world!"));
}
@Test
public void parseBinjrLogsNoFraction() {
assertTrue(parsingTest(BuiltInParsingProfile.ISO, "[2020/07/21-02:56:42] [info] Hello World!"));
assertTrue(parsingTest(BuiltInParsingProfile.ISO, "[2020-11-13 19:59:22] [INFO ] Hello world!"));
}
@Test
public void parseBinjrLogsUnknownSeverity() {
assertTrue(parsingTest(BuiltInParsingProfile.ISO, "[2020-11-13 19:59:22.627] [FOO ] Hello world!"));
}
@Test
public void parseBinjrLogsNoSeverity() {
assertTrue(parsingTest(BuiltInParsingProfile.ISO, "[2020-11-13 19:59:22.627] Hello world!"));
}
@Test
public void parseBinjrStrictFull() {
assertFalse(parsingTest(BuiltInParsingProfile.BINJR_STRICT, "[2020/07/21-02:56:42.460] [info] Hello World!"));
assertTrue(parsingTest(BuiltInParsingProfile.BINJR_STRICT, "[2020-11-13 19:59:22.627] [INFO ] Hello world!"));
}
@Test
public void parseBinjrStrictNoFraction() {
assertFalse(parsingTest(BuiltInParsingProfile.BINJR_STRICT, "[2020/07/21-02:56:42] [info] Hello World!"));
assertFalse(parsingTest(BuiltInParsingProfile.BINJR_STRICT, "[2020-11-13 19:59:22] [INFO ] Hello world!"));
}
@Test
public void parseBinjrStrictUnknownSeverity() {
assertFalse(parsingTest(BuiltInParsingProfile.BINJR_STRICT, "[2020-11-13 19:59:22.627] [FOO ] Hello world!"));
}
@Test
public void parseBinjrStrictNoSeverity() {
assertFalse(parsingTest(BuiltInParsingProfile.BINJR_STRICT, "[2020-11-13 19:59:22.627] Hello world!"));
}
@Test
public void parseGc() {
// Logs with uptime
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.144s][info][gc,phases ] GC(59) Phase 4: Compact heap 11.810ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc,heap ] GC(59) Eden regions: 9->0(130)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc,heap ] GC(59) Survivor regions: 0->0(12)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc,heap ] GC(59) Old regions: 139->137"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc,heap ] GC(59) Archive regions: 0->0"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc,heap ] GC(59) Humongous regions: 10->10"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc,metaspace ] GC(59) Metaspace: 48468K(49200K)->48468K(49200K) NonClass: 41947K(42416K)->41947K(42416K) Class: 6521K(6784K)->6521K(6784K)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.151s][info][gc ] GC(59) Pause Full (System.gc()) 153M->143M(477M) 94.613ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[54.152s][info][gc,cpu ] GC(59) User=0.66s Sys=0.00s Real=0.10s"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[65.250s][info][gc,heap,exit ] Heap"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[65.250s][info][gc,heap,exit ] garbage-first heap total 488448K, used 163531K [0x0000000080000000, 0x0000000100000000)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[65.250s][info][gc,heap,exit ] region size 1024K, 17 young (17408K), 0 survivors (0K)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[65.250s][info][gc,heap,exit ] Metaspace used 48659K, capacity 49344K, committed 49456K, reserved 1091584K"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[65.250s][info][gc,heap,exit ] class space used 6554K, capacity 6778K, committed 6784K, reserved 1048576K"));
//Logs with time (ISO)
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.093+0200][info ][safepoint ] Application time: 0.4142997 seconds"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.094+0200][info ][safepoint ] Entering safepoint region: G1CollectForAllocation"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.097+0200][info ][gc,start ] GC(846) Pause Young (Normal) (GCLocker Initiated GC)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.097+0200][info ][gc,task ] GC(846) Using 143 workers of 143 for evacuation"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,phases ] GC(846) Pre Evacuate Collection Set: 32.2ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,phases ] GC(846) Evacuate Collection Set: 43.3ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,phases ] GC(846) Post Evacuate Collection Set: 9.2ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,phases ] GC(846) Other: 8.9ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,heap ] GC(846) Eden regions: 14->0(949)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,heap ] GC(846) Survivor regions: 10->11(960)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,heap ] GC(846) Old regions: 7420->7420"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,heap ] GC(846) Humongous regions: 11634->11632"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc,metaspace ] GC(846) Metaspace: 231059K(255744K)->231059K(255744K)"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.190+0200][info ][gc ] GC(846) Pause Young (Normal) (GCLocker Initiated GC) 610464M->609992M(614400M) 92.765ms"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.191+0200][info ][gc,cpu ] GC(846) User=4.18s Sys=0.03s Real=0.09s"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.192+0200][info ][safepoint ] Leaving safepoint region"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.192+0200][info ][safepoint ] Total time for which application threads were stopped: 0.0987139 seconds, Stopping threads took: 0.0008103 seconds"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.224+0200][info ][safepoint ] Application time: 0.0321887 seconds"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.286+0200][info ][safepoint ] Entering safepoint region: G1CollectForAllocation"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.292+0200][debug][gc,jni ] Setting _needs_gc. Thread `\"VM Thread\" 62 locked."));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.292+0200][info ][safepoint ] Leaving safepoint region"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.292+0200][info ][safepoint ] Total time for which application threads were stopped: 0.0685306 seconds, Stopping threads took: 0.0622806 seconds"));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.299+0200][debug][gc,jni ] Allocation failed. Thread stalled by JNI critical section. Thread \"ForkJoinPool-280-worker-45_CommitThrottle#commit: top level tasks for transactions\" 48 locked."));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.299+0200][debug][gc,jni ] Allocation failed. Thread stalled by JNI critical section. Thread \"ForkJoinPool-280-worker-111_CommitThrottle#commit: top level tasks for transactions\" 46 locked."));
assertTrue(parsingTest(BuiltInParsingProfile.JVM, "[2023-02-22T06:46:55.305+0200][debug][gc,jni ] Allocation failed. Thread stalled by JNI critical section. Thread \"ForkJoinPool-280-worker-25_CommitThrottle#commit: top level tasks for transactions\" 46 locked."));
}
private boolean parsingTest(ParsingProfile profile, String text) {
var p = new LogEventFormat(profile, ZoneId.systemDefault(), StandardCharsets.UTF_8);
var res = p.parse(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)));
System.out.println(res);
return res.iterator().next() != null;
}
}