Add start scripts

Author: stefanDeveloper

config-test.yaml

@@ -0,0 +1,123 @@
+logging:
+  base:
+    debug: false
+  modules:
+    log_storage.logserver:
+      debug: false
+    log_collection.collector:
+      debug: false
+    log_collection.batch_handler:
+      debug: false
+    log_filtering.prefilter:
+      debug: false
+    data_inspection.inspector:
+      debug: false
+    data_analysis.detector:
+      debug: false
+
+pipeline:
+  log_storage:
+    logserver:
+      input_file: "/opt/file.txt"
+
+
+
+  log_collection:
+    default_batch_handler_config:
+      batch_size: 2000
+      batch_timeout: 30.0
+      subnet_id:
+        ipv4_prefix_length: 24
+        ipv6_prefix_length: 64
+    collectors:
+      - name: "dga_collector"
+        protocol_base: dns
+        required_log_information:
+          - [ "ts", Timestamp, "%Y-%m-%dT%H:%M:%S" ]
+          - [ "status_code", ListItem, [ "NOERROR", "NXDOMAIN" ], [ "NXDOMAIN" ] ]
+          - [ "src_ip", IpAddress ]
+          - [ "dns_server_ip", IpAddress ]
+          - [ "domain_name", RegEx, '^(?=.{1,253}$)((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,63}$' ]
+          - [ "record_type", ListItem, [ "A", "AAAA" ] ]
+          # - [ "response_ip", IpAddress ]
+          - [ "size", RegEx, '^\d+$' ]
+        batch_handler_config_override:
+          batch_timeout: 30.1
+
+  log_filtering:
+    - name: "dga_filter"
+      # method to apply for rule based prefiltering according to the needs
+      relevance_method: check_dga_relevance
+      collector_name: dga_collector
+
+
+  data_inspection:
+    - name: dga_inspector
+      inspector_module_name: "stream_ad_inspector"
+      inspector_class_name: "StreamADInspector"
+      prefilter_name: dga_filter
+      mode: univariate # multivariate, ensemble
+      # Only used when mode is set to ensemble
+      ensemble:
+        model: WeightEnsemble
+        module: streamad.process
+        model_args:
+      models:
+        - model: ZScoreDetector
+          module: streamad.model
+          model_args:
+            is_global: false
+      anomaly_threshold: 0.0001
+      score_threshold: 0.005
+      time_type: ms
+      time_range: 20
+
+  data_analysis:
+    - name: "RF-dga_detector"
+      detector_module_name: "dga_detector"
+      detector_class_name: "DGADetector"
+      model: rf
+      checksum: 5db8bfb617e80361362c33b1d1afc6d762c28e9fa9275fb11514a3bdef76bb88
+      base_url: https://heibox.uni-heidelberg.de/d/0d5cbcbe16cd46a58021/
+      threshold: 0.005
+      inspector_name: dga_inspector
+
+  monitoring:
+    clickhouse_connector:
+      batch_size: 50  # do not set higher
+      batch_timeout: 2.0
+
+
+  zeek:
+    sensors:
+      zeek1:
+        static_analysis: true
+        protocols:
+          - dns
+        interfaces:
+          - enp2s0f0
+
+environment:
+  kafka_brokers:
+    - hostname: localhost
+      internal_port: 8097
+      external_port: 8097
+      node_ip: 127.0.0.1
+    - hostname: localhost
+      internal_port: 8098
+      external_port: 8098
+      node_ip: 127.0.0.1
+    - hostname: localhost
+      internal_port: 8099
+      external_port: 8099
+      node_ip: 127.0.0.1
+  kafka_topics_prefix:
+    pipeline:
+      logserver_in: "pipeline-logserver_in"
+      logserver_to_collector: "pipeline-logserver_to_collector"
+      batch_sender_to_prefilter: "pipeline-batch_sender_to_prefilter"
+      prefilter_to_inspector: "pipeline-prefilter_to_inspector"
+      inspector_to_detector: "pipeline-inspector_to_detector"
+  monitoring:
+    clickhouse_server:
+      hostname: localhost