| 1 | 1 | | package com.nccgroup.loggerplusplus.exports; |
| 2 | 2 | | |
| | 3 | + | import co.elastic.clients.elasticsearch.ElasticsearchClient; |
| | 4 | + | import co.elastic.clients.elasticsearch.core.BulkRequest; |
| | 5 | + | import co.elastic.clients.elasticsearch.core.BulkResponse; |
| | 6 | + | import co.elastic.clients.elasticsearch.core.IndexRequest; |
| | 7 | + | import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; |
| | 8 | + | import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; |
| | 9 | + | import co.elastic.clients.elasticsearch.indices.ExistsRequest; |
| | 10 | + | import co.elastic.clients.elasticsearch.indices.GetIndexRequest; |
| | 11 | + | import co.elastic.clients.json.JsonData; |
| | 12 | + | import co.elastic.clients.json.jackson.JacksonJsonpGenerator; |
| | 13 | + | import co.elastic.clients.json.jackson.JacksonJsonpMapper; |
| | 14 | + | import co.elastic.clients.transport.ElasticsearchTransport; |
| | 15 | + | import co.elastic.clients.transport.endpoints.BooleanResponse; |
| | 16 | + | import co.elastic.clients.transport.rest_client.RestClientTransport; |
| 3 | 17 | | import com.coreyd97.BurpExtenderUtilities.Preferences; |
| | 18 | + | import com.google.gson.Gson; |
| | 19 | + | import com.google.gson.JsonObject; |
| 4 | 20 | | import com.nccgroup.loggerplusplus.LoggerPlusPlus; |
| 5 | 21 | | import com.nccgroup.loggerplusplus.filter.logfilter.LogTableFilter; |
| 6 | 22 | | import com.nccgroup.loggerplusplus.filter.parser.ParseException; |
| skipped 8 lines |
| 15 | 31 | | import org.apache.http.message.BasicHeader; |
| 16 | 32 | | import org.apache.logging.log4j.LogManager; |
| 17 | 33 | | import org.apache.logging.log4j.Logger; |
| 18 | | - | import org.elasticsearch.action.bulk.BulkItemResponse; |
| 19 | | - | import org.elasticsearch.action.bulk.BulkRequest; |
| 20 | | - | import org.elasticsearch.action.bulk.BulkResponse; |
| 21 | | - | import org.elasticsearch.action.index.IndexRequest; |
| 22 | | - | import org.elasticsearch.client.RequestOptions; |
| 23 | 34 | | import org.elasticsearch.client.RestClient; |
| 24 | 35 | | import org.elasticsearch.client.RestClientBuilder; |
| 25 | | - | import org.elasticsearch.client.RestHighLevelClient; |
| 26 | | - | import org.elasticsearch.client.indices.CreateIndexRequest; |
| 27 | | - | import org.elasticsearch.client.indices.GetIndexRequest; |
| 28 | | - | import org.elasticsearch.common.xcontent.XContentBuilder; |
| 29 | | - | import org.elasticsearch.common.xcontent.XContentElasticsearchExtension; |
| | 36 | + | |
| 30 | 37 | | |
| 31 | 38 | | import javax.swing.*; |
| 32 | 39 | | import java.io.IOException; |
| skipped 6 lines |
| 39 | 46 | | import java.util.concurrent.ScheduledFuture; |
| 40 | 47 | | import java.util.concurrent.TimeUnit; |
| 41 | 48 | | |
| 42 | | - | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; |
| 43 | | - | |
| 44 | 49 | | @Log4j2 |
| 45 | 50 | | public class ElasticExporter extends AutomaticLogExporter implements ExportPanelProvider, ContextMenuExportProvider { |
| 46 | 51 | | |
| 47 | | - | RestHighLevelClient httpClient; |
| | 52 | + | ElasticsearchClient elasticClient; |
| 48 | 53 | | ArrayList<LogEntry> pendingEntries; |
| 49 | 54 | | LogTableFilter logFilter; |
| 50 | 55 | | private List<LogEntryField> fields; |
| skipped 3 lines |
| 54 | 59 | | |
| 55 | 60 | | private final ScheduledExecutorService executorService; |
| 56 | 61 | | private final ElasticExporterControlPanel controlPanel; |
| | 62 | + | private final Gson gson; |
| 57 | 63 | | |
| 58 | 64 | | private Logger logger = LogManager.getLogger(this); |
| 59 | 65 | | |
| 60 | 66 | | protected ElasticExporter(ExportController exportController, Preferences preferences) { |
| 61 | 67 | | super(exportController, preferences); |
| 62 | 68 | | this.fields = new ArrayList<>(preferences.getSetting(Globals.PREF_PREVIOUS_ELASTIC_FIELDS)); |
| | 69 | + | this.gson = LoggerPlusPlus.gsonProvider.getGson(); |
| 63 | 70 | | executorService = Executors.newScheduledThreadPool(1); |
| 64 | 71 | | |
| 65 | 72 | | if ((boolean) preferences.getSetting(Globals.PREF_ELASTIC_AUTOSTART_GLOBAL) |
| skipped 41 lines |
| 107 | 114 | | int port = preferences.getSetting(Globals.PREF_ELASTIC_PORT); |
| 108 | 115 | | indexName = preferences.getSetting(Globals.PREF_ELASTIC_INDEX); |
| 109 | 116 | | String protocol = preferences.getSetting(Globals.PREF_ELASTIC_PROTOCOL).toString(); |
| 110 | | - | RestClientBuilder builder = RestClient.builder(new HttpHost(address, port, protocol)); |
| | 117 | + | RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(address, port, protocol)); |
| 111 | 118 | | logger.info(String.format("Starting ElasticSearch exporter. %s://%s:%s/%s", protocol, address, port, indexName)); |
| 112 | 119 | | |
| 113 | 120 | | Globals.ElasticAuthType authType = preferences.getSetting(Globals.PREF_ELASTIC_AUTH); |
| skipped 15 lines |
| 129 | 136 | | if (!"".equals(user) && !"".equalsIgnoreCase(pass)) { |
| 130 | 137 | | logger.info(String.format("ElasticSearch using %s, Username: %s", authType, user)); |
| 131 | 138 | | String authValue = Base64.getEncoder().encodeToString((user + ":" + pass).getBytes(StandardCharsets.UTF_8)); |
| 132 | | - | builder.setDefaultHeaders(new Header[]{new BasicHeader("Authorization", String.format("%s %s", authType, authValue))}); |
| | 139 | + | restClientBuilder.setDefaultHeaders(new Header[]{new BasicHeader("Authorization", String.format("%s %s", authType, authValue))}); |
| 133 | 140 | | } |
| 134 | 141 | | |
| 135 | | - | httpClient = new RestHighLevelClient(builder); |
| | 142 | + | |
| | 143 | + | ElasticsearchTransport transport = new RestClientTransport(restClientBuilder.build(), new JacksonJsonpMapper()); |
| | 144 | + | |
| | 145 | + | elasticClient = new ElasticsearchClient(transport); |
| 136 | 146 | | |
| 137 | 147 | | createIndices(); |
| 138 | 148 | | pendingEntries = new ArrayList<>(); |
| skipped 36 lines |
| 175 | 185 | | } |
| 176 | 186 | | |
| 177 | 187 | | private void createIndices() throws IOException { |
| 178 | | - | GetIndexRequest request = new GetIndexRequest(this.indexName); |
| | 188 | + | ExistsRequest existsRequest = new ExistsRequest.Builder().index(this.indexName).build(); |
| 179 | 189 | | |
| 180 | | - | boolean exists = httpClient.indices().exists(request, RequestOptions.DEFAULT); |
| | 190 | + | BooleanResponse exists = elasticClient.indices().exists(existsRequest); |
| 181 | 191 | | |
| 182 | | - | if(!exists) { |
| 183 | | - | CreateIndexRequest _request = new CreateIndexRequest(this.indexName); |
| 184 | | - | httpClient.indices().create(_request, RequestOptions.DEFAULT); |
| | 192 | + | if(!exists.value()) { |
| | 193 | + | CreateIndexRequest createIndexRequest = new CreateIndexRequest.Builder().index(this.indexName).build(); |
| | 194 | + | elasticClient.indices().create(createIndexRequest); |
| 185 | 195 | | } |
| 186 | 196 | | } |
| 187 | 197 | | |
| 188 | | - | public IndexRequest buildIndexRequest(LogEntry logEntry) throws IOException { |
| 189 | | - | XContentBuilder builder = jsonBuilder().startObject(); |
| | 198 | + | public JsonObject serializeLogEntry(LogEntry logEntry) { |
| | 199 | + | //Todo Better serialization of entries |
| | 200 | + | JsonObject jsonObject = new JsonObject(); |
| 190 | 201 | | for (LogEntryField field : this.fields) { |
| 191 | 202 | | Object value = formatValue(logEntry.getValueByKey(field)); |
| 192 | 203 | | try { |
| 193 | | - | //For some reason, the XContentElasticsearchExtension service cannot be loaded |
| 194 | | - | //when in burp, so we must format dates manually ourselves :( |
| 195 | | - | //TODO investigate further |
| 196 | | - | if (value instanceof Date) { |
| 197 | | - | builder.field(field.getFullLabel(), XContentElasticsearchExtension.DEFAULT_DATE_PRINTER.print(((Date) value).getTime())); |
| 198 | | - | } else { |
| 199 | | - | builder.field(field.getFullLabel(), value); |
| 200 | | - | } |
| | 204 | + | jsonObject.addProperty(field.getFullLabel(), gson.toJson(value)); |
| 201 | 205 | | }catch (Exception e){ |
| 202 | 206 | | log.error("ElasticExporter: " + value); |
| 203 | 207 | | log.error("ElasticExporter: " + e.getMessage()); |
| 204 | 208 | | throw e; |
| 205 | 209 | | } |
| 206 | 210 | | } |
| 207 | | - | builder.endObject(); |
| 208 | | - | |
| 209 | | - | return new IndexRequest(this.indexName, "doc").source(builder); //TODO Remove deprecated ES6 methods. |
| | 211 | + | return jsonObject; |
| 210 | 212 | | } |
| 211 | 213 | | |
| 212 | 214 | | private void indexPendingEntries(){ |
| 213 | 215 | | try { |
| 214 | 216 | | if (this.pendingEntries.size() == 0) return; |
| 215 | 217 | | |
| 216 | | - | BulkRequest httpBulkBuilder = new BulkRequest(); |
| | 218 | + | BulkRequest.Builder bulkBuilder = new BulkRequest.Builder(); |
| 217 | 219 | | |
| 218 | 220 | | ArrayList<LogEntry> entriesInBulk; |
| 219 | 221 | | synchronized (pendingEntries) { |
| skipped 3 lines |
| 223 | 225 | | |
| 224 | 226 | | for (LogEntry logEntry : entriesInBulk) { |
| 225 | 227 | | try { |
| 226 | | - | IndexRequest request = buildIndexRequest(logEntry); |
| 227 | | - | httpBulkBuilder.add(request); |
| | 228 | + | bulkBuilder.operations(op -> op |
| | 229 | + | .index(idx -> idx |
| | 230 | + | .index(this.indexName) |
| | 231 | + | .document(serializeLogEntry(logEntry)) |
| | 232 | + | ) |
| | 233 | + | ); |
| | 234 | + | |
| 228 | 235 | | } catch (Exception e) { |
| 229 | 236 | | log.error("Could not build elastic export request for entry: " + e.getMessage()); |
| 230 | 237 | | //Could not build index request. Ignore it? |
| skipped 1 lines |
| 232 | 239 | | } |
| 233 | 240 | | |
| 234 | 241 | | try { |
| 235 | | - | BulkResponse bulkResponse = httpClient.bulk(httpBulkBuilder, RequestOptions.DEFAULT); |
| 236 | | - | if (bulkResponse.hasFailures()) { |
| 237 | | - | for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) { |
| 238 | | - | log.error(bulkItemResponse.getFailureMessage()); |
| | 242 | + | BulkResponse bulkResponse = elasticClient.bulk(bulkBuilder.build()); |
| | 243 | + | if (bulkResponse.errors()) { |
| | 244 | + | for (BulkResponseItem bulkResponseItem : bulkResponse.items()) { |
| | 245 | + | log.error(bulkResponseItem.error().reason()); |
| 239 | 246 | | } |
| 240 | 247 | | } |
| 241 | 248 | | connectFailedCounter = 0; |
| skipped 35 lines |
