crash.software
Projects
Pull Requests
Issues
Builds
bearer
Code
Files
Commits
Branches
Tags
Pull Requests
Code Comments
Code Compare
Issues
Builds
Statistics
Child Projects
‹cra.sh›
RESTful API
Projects STRLCPY bearer Commits 480b7466
🤬

  • fix: performance on large repositories (#220)

    * wip: ignore symlink and extend to bigger files

* wip: improve processor efficiency

* refactor: improve performance of processors

* refactor: cleanup

* fix: use correct encrypted field module name and update snapshots

Co-authored-by: David Roe <[email protected]>
  • Loading...
  • Cédric Fabianski committed with GitHub 1 year ago
    480b7466
    1 parent cf89a530
Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_cookies
    skipped 43 lines
    44 44   
    45 45   
    46 46  --
     47 +Processing Detectors
     48 +Finished processing Detectors
     49 +Processing Dataflow
     50 +Finished processing Dataflow
    47 51   
    48 52   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_jwt
    skipped 33 lines
    34 34   
    35 35   
    36 36  --
     37 +Processing Detectors
     38 +Finished processing Detectors
     39 +Processing Dataflow
     40 +Finished processing Dataflow
    37 41   
    38 42   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-detect_rails_session
    skipped 25 lines
    26 26   
    27 27   
    28 28  --
     29 +Processing Detectors
     30 +Finished processing Detectors
     31 +Processing Dataflow
     32 +Finished processing Dataflow
    29 33   
    30 34   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-detect_ruby_logger
    skipped 43 lines
    44 44   
    45 45   
    46 46  --
     47 +Processing Detectors
     48 +Finished processing Detectors
     49 +Processing Dataflow
     50 +Finished processing Dataflow
    47 51   
    48 52   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-ftp
    skipped 84 lines
    85 85   
    86 86   
    87 87  --
     88 +Processing Detectors
     89 +Finished processing Detectors
     90 +Processing Dataflow
     91 +Finished processing Dataflow
    88 92   
    89 93   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_file_detection
    skipped 172 lines
    173 173   
    174 174   
    175 175  --
     176 +Processing Detectors
     177 +Finished processing Detectors
     178 +Processing Dataflow
     179 +Finished processing Dataflow
    176 180   
    177 181   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-ruby_http_detection
    skipped 474 lines
    475 475   
    476 476   
    477 477  --
     478 +Processing Detectors
     479 +Finished processing Detectors
     480 +Processing Dataflow
     481 +Finished processing Dataflow
    478 482   
    479 483   
  • ■ ■ ■ ■ ■ ■
    integration/custom_detectors/.snapshots/TestCustomDetectors-ssl_certificate_verification_disabled
    skipped 29 lines
    30 30   
    31 31   
    32 32  --
     33 +Processing Detectors
     34 +Finished processing Detectors
     35 +Processing Dataflow
     36 +Finished processing Dataflow
    33 37   
    34 38   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestInitCommand-init
    skipped 13 lines
    14 14   languages:
    15 15   - ruby
    16 16   param_parenting: true
    17  - processors: []
    18 17   patterns:
    19 18   - pattern: |
    20 19   class $CLASS_NAME < ApplicationRecord
    skipped 12 lines
    33 32   languages:
    34 33   - ruby
    35 34   param_parenting: false
    36  - processors: []
    37 35   patterns:
    38 36   - pattern: |
    39 37   cookies[...] = $ANYTHING
    skipped 30 lines
    70 68   languages:
    71 69   - ruby
    72 70   param_parenting: false
    73  - processors: []
    74 71   patterns:
    75 72   - pattern: |
    76 73   Rails.application.configure do
    skipped 12 lines
    89 86   languages:
    90 87   - ruby
    91 88   param_parenting: false
    92  - processors: []
    93 89   patterns:
    94 90   - pattern: |
    95 91   Net::FTP.new()
    skipped 13 lines
    109 105   languages:
    110 106   - ruby
    111 107   param_parenting: false
    112  - processors: []
    113 108   patterns:
    114 109   - pattern: |
    115 110   Net::FTP.open do
    skipped 12 lines
    128 123   languages:
    129 124   - ruby
    130 125   param_parenting: false
    131  - processors: []
    132 126   patterns:
    133 127   - pattern: |
    134 128   Rails.application.configure do
    skipped 21 lines
    156 150   languages:
    157 151   - ruby
    158 152   param_parenting: false
    159  - processors: []
    160 153   patterns:
    161 154   - pattern: |
    162 155   JWT.encode(<$ARGUMENT>)
    skipped 10 lines
    173 166   languages:
    174 167   - ruby
    175 168   param_parenting: false
    176  - processors: []
    177 169   patterns:
    178 170   - pattern: |
    179 171   session[...] = $ANYTHING
    skipped 10 lines
    190 182   languages:
    191 183   - ruby
    192 184   param_parenting: false
    193  - processors: []
    194 185   patterns:
    195 186   - pattern: |
    196 187   logger.info(<$ARGUMENT>)
    skipped 13 lines
    210 201   languages:
    211 202   - ruby
    212 203   param_parenting: false
    213  - processors: []
    214 204   patterns:
    215 205   - pattern: |
    216 206   Sentry::Breadcrumb.new(<$ARGUMENT>)
    skipped 21 lines
    238 228   languages:
    239 229   - sql
    240 230   param_parenting: true
    241  - processors:
    242  - - query: |
    243  - verified_by = data.bearer.encrypted_verified.verified_by
    244  - encrypted = data.bearer.encrypted_verified.encrypted
    245  - modules:
    246  - - name: bearer.encrypted_verified
    247  - content: |-
    248  - package bearer.encrypted_verified
    249  - 
    250  - import future.keywords
    251  - 
    252  - 
    253  - default encrypted := false
    254  - 
    255  - 
    256  - ruby_encrypted[location] {
    257  - some detection in input.all_detections
    258  - detection.detector_type == "detect_encrypted_ruby_class_properties"
    259  - detection.value.classification.decision.state == "valid"
    260  - location = detection
    261  - }
    262  - 
    263  - encrypted = true {
    264  - some detection in ruby_encrypted
    265  - detection.value.object_name == input.target.value.object_name
    266  - detection.value.field_name == input.target.value.field_name
    267  - input.target.value.field_name != ""
    268  - input.target.value.object_name != ""
    269  - }
    270  - 
    271  - verified_by[verification] {
    272  - some detection in ruby_encrypted
    273  - detection.value.object_name == input.target.value.object_name
    274  - detection.value.field_name == input.target.value.field_name
    275  - 
    276  - verification = {
    277  - "detector": "detect_encrypted_ruby_class_properties",
    278  - "filename": detection.source.filename,
    279  - "line_number": detection.source.line_number
    280  - }
    281  - }
    282 231   patterns:
    283 232   - pattern: |
    284 233   CREATE TABLE public.$TABLE_NAME (
    skipped 17 lines
    302 251   languages:
    303 252   - ruby
    304 253   param_parenting: true
    305  - processors: []
    306 254   patterns:
    307 255   - pattern: |
    308 256   CSV.generate { <$DATA_TYPE> }
    skipped 24 lines
    333 281   languages:
    334 282   - ruby
    335 283   param_parenting: false
    336  - processors: []
    337 284   patterns:
    338 285   - pattern: |
    339 286   URI.encode_www_form(<$DATA_TYPE>)
    skipped 26 lines
    366 313   languages:
    367 314   - ruby
    368 315   param_parenting: false
    369  - processors: []
    370 316   patterns:
    371 317   - pattern: |
    372 318   URI(<$INSECURE_URL>)
    skipped 23 lines
    396 342   languages:
    397 343   - ruby
    398 344   param_parenting: false
    399  - processors: []
    400 345   patterns:
    401 346   - pattern: |
    402 347   URI.encode_www_form(<$DATA_TYPE>)
    skipped 26 lines
    429 374   languages:
    430 375   - ruby
    431 376   param_parenting: false
    432  - processors: []
    433 377   patterns:
    434 378   - pattern: |
    435 379   Net::HTTP.post_form(<$INSECURE_URL>)
    skipped 23 lines
    459 403   languages:
    460 404   - ruby
    461 405   param_parenting: false
    462  - processors: []
    463 406   patterns:
    464 407   - pattern: |
    465 408   Net::HTTP.start($_, $_, $_, :verify_mode => OpenSSL::SSL::VERIFY_NONE) do
    skipped 1056 lines
    1522 1465   skip-path: []
    1523 1466  worker:
    1524 1467   existing-worker: ""
    1525  - file-size-max: 100000
     1468 + file-size-max: 2000000
    1526 1469   files-to-batch: 1
    1527 1470   memory-max: 800000000
    1528 1471   timeout: 10m0s
    skipped 7 lines
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestMetadataFlags-help-scan
    skipped 23 lines
    24 24   
    25 25  Worker Flags
    26 26   --existing-worker string Specify the URL of an existing worker.
    27  - --file-size-max int Ignore files larger than the specified value. (default 100000)
     27 + --file-size-max int Ignore files larger than the specified value. (default 2000000)
    28 28   --files-to-batch int Specify the number of files to batch per worker. (default 1)
    29 29   --memory-max int If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
    30 30   --timeout duration The maximum time alloted to complete the scan. (default 10m0s)
    skipped 18 lines
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestMetadataFlags-scan-help
    skipped 23 lines
    24 24   
    25 25  Worker Flags
    26 26   --existing-worker string Specify the URL of an existing worker.
    27  - --file-size-max int Ignore files larger than the specified value. (default 100000)
     27 + --file-size-max int Ignore files larger than the specified value. (default 2000000)
    28 28   --files-to-batch int Specify the number of files to batch per worker. (default 1)
    29 29   --memory-max int If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
    30 30   --timeout duration The maximum time alloted to complete the scan. (default 10m0s)
    skipped 18 lines
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-domain-resolution-disabled
    1 1  [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
    2 2   
    3 3  --
     4 +Processing Detectors
     5 +Finished processing Detectors
    4 6   
    5 7   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-format-json
    1 1  [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
    2 2   
    3 3  --
     4 +Processing Detectors
     5 +Finished processing Detectors
    4 6   
    5 7   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-format-yaml
    skipped 63 lines
    64 64   
    65 65   
    66 66  --
     67 +Processing Detectors
     68 +Finished processing Detectors
    67 69   
    68 70   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-health-context
    1 1  [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
    2 2   
    3 3  --
     4 +Processing Detectors
     5 +Finished processing Detectors
    4 6   
    5 7   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-report-dataflow
    1 1  {"data_types":[{"name":"Email Address","detectors":[{"name":"ruby","locations":[{"filename":"testdata/simple/main.rb","line_number":1}]}]}],"risks":[{"detector_id":"detect_ruby_logger","data_types":[{"name":"Email Address","stored":false,"locations":[{"filename":"testdata/simple/main.rb","line_number":1,"parent":{"line_number":1,"content":"logger.info(\"user info\", user.email)"}}]}]}],"components":[]}
    2 2   
    3 3  --
     4 +Processing Detectors
     5 +Finished processing Detectors
     6 +Processing Dataflow
     7 +Finished processing Dataflow
    4 8   
    5 9   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-report-dataflow-verified-by
    skipped 143 lines
    144 144   
    145 145   
    146 146  --
     147 +Processing Detectors
     148 +Finished processing Detectors
     149 +Processing Dataflow
     150 +Finished processing Dataflow
    147 151   
    148 152   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-report-detectors
    1 1  [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
    2 2   
    3 3  --
     4 +Processing Detectors
     5 +Finished processing Detectors
    4 6   
    5 7   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-report-policies
    skipped 10 lines
    11 11   
    12 12   
    13 13  --
     14 +Processing Detectors
     15 +Finished processing Detectors
     16 +Processing Dataflow
     17 +Finished processing Dataflow
     18 +Processing Policies
     19 +Processing policy Application level encryption missing
     20 +Finished processing policy Application level encryption missing
     21 +Processing policy Cookie leaking
     22 +Finished processing policy Cookie leaking
     23 +Processing policy HTTP GET parameters
     24 +Finished processing policy HTTP GET parameters
     25 +Processing policy Insecure communication
     26 +Finished processing policy Insecure communication
     27 +Processing policy Insecure FTP
     28 +Finished processing policy Insecure FTP
     29 +Processing policy Insecure FTP with Data Category
     30 +Finished processing policy Insecure FTP with Data Category
     31 +Processing policy Insecure HTTP GET
     32 +Finished processing policy Insecure HTTP GET
     33 +Processing policy Insecure HTTP with Data Category
     34 +Finished processing policy Insecure HTTP with Data Category
     35 +Processing policy Insecure SMTP
     36 +Finished processing policy Insecure SMTP
     37 +Processing policy JWT leaking
     38 +Finished processing policy JWT leaking
     39 +Processing policy Logger leaking
     40 +Finished processing policy Logger leaking
     41 +Processing policy Third-party data category exposure
     42 +Finished processing policy Third-party data category exposure
     43 +Processing policy Session leaking
     44 +Finished processing policy Session leaking
     45 +Processing policy SSL certificate verification disabled
     46 +Finished processing policy SSL certificate verification disabled
     47 +Finished processing policies
    14 48   
    15 49   
  • ■ ■ ■ ■ ■ ■
    integration/flags/.snapshots/TestReportFlags-skipped-paths
    1 1  [{"detector_type":"detect_ruby_logger","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"custom_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user","parent":{"content":"logger.info(\"user info\", user.email)","line_number":1}}},{"detector_type":"ruby","source":{"column_number":8,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"field_name":"info","field_type":"","field_type_simple":"unknown","object_name":"logger"}},{"detector_type":"ruby","source":{"column_number":31,"filename":"main.rb","language":"Ruby","language_type":"programming","line_number":1,"text":null},"type":"schema_classified","value":{"classification":{"data_type":{"category_uuid":"cef587dd-76db-430b-9e18-7b031e1a193b","name":"Email Address","uuid":"22e24c62-82d3-4b72-827c-e261533331bd"},"decision":{"reason":"known_pattern","state":"valid"},"name":"email"},"field_name":"email","field_type":"","field_type_simple":"unknown","object_name":"user"}}]
    2 2   
    3 3  --
     4 +Processing Detectors
     5 +Finished processing Detectors
    4 6   
    5 7   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicesWithHealthContext-logger_leaking
    skipped 11 lines
    12 12   
    13 13   
    14 14  --
     15 +Processing Detectors
     16 +Finished processing Detectors
     17 +Processing Dataflow
     18 +Finished processing Dataflow
     19 +Processing Policies
     20 +Processing policy Application level encryption missing
     21 +Finished processing policy Application level encryption missing
     22 +Processing policy Cookie leaking
     23 +Finished processing policy Cookie leaking
     24 +Processing policy HTTP GET parameters
     25 +Finished processing policy HTTP GET parameters
     26 +Processing policy Insecure communication
     27 +Finished processing policy Insecure communication
     28 +Processing policy Insecure FTP
     29 +Finished processing policy Insecure FTP
     30 +Processing policy Insecure FTP with Data Category
     31 +Finished processing policy Insecure FTP with Data Category
     32 +Processing policy Insecure HTTP GET
     33 +Finished processing policy Insecure HTTP GET
     34 +Processing policy Insecure HTTP with Data Category
     35 +Finished processing policy Insecure HTTP with Data Category
     36 +Processing policy Insecure SMTP
     37 +Finished processing policy Insecure SMTP
     38 +Processing policy JWT leaking
     39 +Finished processing policy JWT leaking
     40 +Processing policy Logger leaking
     41 +Finished processing policy Logger leaking
     42 +Processing policy Third-party data category exposure
     43 +Finished processing policy Third-party data category exposure
     44 +Processing policy Session leaking
     45 +Finished processing policy Session leaking
     46 +Processing policy SSL certificate verification disabled
     47 +Finished processing policy SSL certificate verification disabled
     48 +Finished processing policies
    15 49   
    16 50   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicesWithHealthContext-sending_data_in_category_to_third_party
    skipped 42 lines
    43 43   
    44 44   
    45 45  --
     46 +Processing Detectors
     47 +Finished processing Detectors
     48 +Processing Dataflow
     49 +Finished processing Dataflow
     50 +Processing Policies
     51 +Processing policy Application level encryption missing
     52 +Finished processing policy Application level encryption missing
     53 +Processing policy Cookie leaking
     54 +Finished processing policy Cookie leaking
     55 +Processing policy HTTP GET parameters
     56 +Finished processing policy HTTP GET parameters
     57 +Processing policy Insecure communication
     58 +Finished processing policy Insecure communication
     59 +Processing policy Insecure FTP
     60 +Finished processing policy Insecure FTP
     61 +Processing policy Insecure FTP with Data Category
     62 +Finished processing policy Insecure FTP with Data Category
     63 +Processing policy Insecure HTTP GET
     64 +Finished processing policy Insecure HTTP GET
     65 +Processing policy Insecure HTTP with Data Category
     66 +Finished processing policy Insecure HTTP with Data Category
     67 +Processing policy Insecure SMTP
     68 +Finished processing policy Insecure SMTP
     69 +Processing policy JWT leaking
     70 +Finished processing policy JWT leaking
     71 +Processing policy Logger leaking
     72 +Finished processing policy Logger leaking
     73 +Processing policy Third-party data category exposure
     74 +Finished processing policy Third-party data category exposure
     75 +Processing policy Session leaking
     76 +Finished processing policy Session leaking
     77 +Processing policy SSL certificate verification disabled
     78 +Finished processing policy SSL certificate verification disabled
     79 +Finished processing policies
    46 80   
    47 81   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-application_level_encryption_missing_schema_rb
    skipped 49 lines
    50 50   
    51 51   
    52 52  --
     53 +Processing Detectors
     54 +Finished processing Detectors
     55 +Processing Dataflow
     56 +Finished processing Dataflow
     57 +Processing Policies
     58 +Processing policy Application level encryption missing
     59 +Finished processing policy Application level encryption missing
     60 +Processing policy Cookie leaking
     61 +Finished processing policy Cookie leaking
     62 +Processing policy HTTP GET parameters
     63 +Finished processing policy HTTP GET parameters
     64 +Processing policy Insecure communication
     65 +Finished processing policy Insecure communication
     66 +Processing policy Insecure FTP
     67 +Finished processing policy Insecure FTP
     68 +Processing policy Insecure FTP with Data Category
     69 +Finished processing policy Insecure FTP with Data Category
     70 +Processing policy Insecure HTTP GET
     71 +Finished processing policy Insecure HTTP GET
     72 +Processing policy Insecure HTTP with Data Category
     73 +Finished processing policy Insecure HTTP with Data Category
     74 +Processing policy Insecure SMTP
     75 +Finished processing policy Insecure SMTP
     76 +Processing policy JWT leaking
     77 +Finished processing policy JWT leaking
     78 +Processing policy Logger leaking
     79 +Finished processing policy Logger leaking
     80 +Processing policy Third-party data category exposure
     81 +Finished processing policy Third-party data category exposure
     82 +Processing policy Session leaking
     83 +Finished processing policy Session leaking
     84 +Processing policy SSL certificate verification disabled
     85 +Finished processing policy SSL certificate verification disabled
     86 +Finished processing policies
    53 87   
    54 88   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-application_level_encryption_missing_structure_sql
    skipped 35 lines
    36 36   
    37 37   
    38 38  --
     39 +Processing Detectors
     40 +Finished processing Detectors
     41 +Processing Dataflow
     42 +Finished processing Dataflow
     43 +Processing Policies
     44 +Processing policy Application level encryption missing
     45 +Finished processing policy Application level encryption missing
     46 +Processing policy Cookie leaking
     47 +Finished processing policy Cookie leaking
     48 +Processing policy HTTP GET parameters
     49 +Finished processing policy HTTP GET parameters
     50 +Processing policy Insecure communication
     51 +Finished processing policy Insecure communication
     52 +Processing policy Insecure FTP
     53 +Finished processing policy Insecure FTP
     54 +Processing policy Insecure FTP with Data Category
     55 +Finished processing policy Insecure FTP with Data Category
     56 +Processing policy Insecure HTTP GET
     57 +Finished processing policy Insecure HTTP GET
     58 +Processing policy Insecure HTTP with Data Category
     59 +Finished processing policy Insecure HTTP with Data Category
     60 +Processing policy Insecure SMTP
     61 +Finished processing policy Insecure SMTP
     62 +Processing policy JWT leaking
     63 +Finished processing policy JWT leaking
     64 +Processing policy Logger leaking
     65 +Finished processing policy Logger leaking
     66 +Processing policy Third-party data category exposure
     67 +Finished processing policy Third-party data category exposure
     68 +Processing policy Session leaking
     69 +Finished processing policy Session leaking
     70 +Processing policy SSL certificate verification disabled
     71 +Finished processing policy SSL certificate verification disabled
     72 +Finished processing policies
    39 73   
    40 74   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-http
    skipped 1 lines
    2 2   
    3 3   
    4 4  --
     5 +Processing Detectors
     6 +Finished processing Detectors
     7 +Processing Dataflow
     8 +Finished processing Dataflow
     9 +Processing Policies
     10 +Processing policy Application level encryption missing
     11 +Finished processing policy Application level encryption missing
     12 +Processing policy Cookie leaking
     13 +Finished processing policy Cookie leaking
     14 +Processing policy HTTP GET parameters
     15 +Finished processing policy HTTP GET parameters
     16 +Processing policy Insecure communication
     17 +Finished processing policy Insecure communication
     18 +Processing policy Insecure FTP
     19 +Finished processing policy Insecure FTP
     20 +Processing policy Insecure FTP with Data Category
     21 +Finished processing policy Insecure FTP with Data Category
     22 +Processing policy Insecure HTTP GET
     23 +Finished processing policy Insecure HTTP GET
     24 +Processing policy Insecure HTTP with Data Category
     25 +Finished processing policy Insecure HTTP with Data Category
     26 +Processing policy Insecure SMTP
     27 +Finished processing policy Insecure SMTP
     28 +Processing policy JWT leaking
     29 +Finished processing policy JWT leaking
     30 +Processing policy Logger leaking
     31 +Finished processing policy Logger leaking
     32 +Processing policy Third-party data category exposure
     33 +Finished processing policy Third-party data category exposure
     34 +Processing policy Session leaking
     35 +Finished processing policy Session leaking
     36 +Processing policy SSL certificate verification disabled
     37 +Finished processing policy SSL certificate verification disabled
     38 +Finished processing policies
    5 39   
    6 40   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-insecure_communication
    skipped 13 lines
    14 14   
    15 15   
    16 16  --
     17 +Processing Detectors
     18 +Finished processing Detectors
     19 +Processing Dataflow
     20 +Finished processing Dataflow
     21 +Processing Policies
     22 +Processing policy Application level encryption missing
     23 +Finished processing policy Application level encryption missing
     24 +Processing policy Cookie leaking
     25 +Finished processing policy Cookie leaking
     26 +Processing policy HTTP GET parameters
     27 +Finished processing policy HTTP GET parameters
     28 +Processing policy Insecure communication
     29 +Finished processing policy Insecure communication
     30 +Processing policy Insecure FTP
     31 +Finished processing policy Insecure FTP
     32 +Processing policy Insecure FTP with Data Category
     33 +Finished processing policy Insecure FTP with Data Category
     34 +Processing policy Insecure HTTP GET
     35 +Finished processing policy Insecure HTTP GET
     36 +Processing policy Insecure HTTP with Data Category
     37 +Finished processing policy Insecure HTTP with Data Category
     38 +Processing policy Insecure SMTP
     39 +Finished processing policy Insecure SMTP
     40 +Processing policy JWT leaking
     41 +Finished processing policy JWT leaking
     42 +Processing policy Logger leaking
     43 +Finished processing policy Logger leaking
     44 +Processing policy Third-party data category exposure
     45 +Finished processing policy Third-party data category exposure
     46 +Processing policy Session leaking
     47 +Finished processing policy Session leaking
     48 +Processing policy SSL certificate verification disabled
     49 +Finished processing policy SSL certificate verification disabled
     50 +Finished processing policies
    17 51   
    18 52   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-insecure_ftp
    skipped 70 lines
    71 71   
    72 72   
    73 73  --
     74 +Processing Detectors
     75 +Finished processing Detectors
     76 +Processing Dataflow
     77 +Finished processing Dataflow
     78 +Processing Policies
     79 +Processing policy Application level encryption missing
     80 +Finished processing policy Application level encryption missing
     81 +Processing policy Cookie leaking
     82 +Finished processing policy Cookie leaking
     83 +Processing policy HTTP GET parameters
     84 +Finished processing policy HTTP GET parameters
     85 +Processing policy Insecure communication
     86 +Finished processing policy Insecure communication
     87 +Processing policy Insecure FTP
     88 +Finished processing policy Insecure FTP
     89 +Processing policy Insecure FTP with Data Category
     90 +Finished processing policy Insecure FTP with Data Category
     91 +Processing policy Insecure HTTP GET
     92 +Finished processing policy Insecure HTTP GET
     93 +Processing policy Insecure HTTP with Data Category
     94 +Finished processing policy Insecure HTTP with Data Category
     95 +Processing policy Insecure SMTP
     96 +Finished processing policy Insecure SMTP
     97 +Processing policy JWT leaking
     98 +Finished processing policy JWT leaking
     99 +Processing policy Logger leaking
     100 +Finished processing policy Logger leaking
     101 +Processing policy Third-party data category exposure
     102 +Finished processing policy Third-party data category exposure
     103 +Processing policy Session leaking
     104 +Finished processing policy Session leaking
     105 +Processing policy SSL certificate verification disabled
     106 +Finished processing policy SSL certificate verification disabled
     107 +Finished processing policies
    74 108   
    75 109   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-insecure_smtp
    skipped 29 lines
    30 30   
    31 31   
    32 32  --
     33 +Processing Detectors
     34 +Finished processing Detectors
     35 +Processing Dataflow
     36 +Finished processing Dataflow
     37 +Processing Policies
     38 +Processing policy Application level encryption missing
     39 +Finished processing policy Application level encryption missing
     40 +Processing policy Cookie leaking
     41 +Finished processing policy Cookie leaking
     42 +Processing policy HTTP GET parameters
     43 +Finished processing policy HTTP GET parameters
     44 +Processing policy Insecure communication
     45 +Finished processing policy Insecure communication
     46 +Processing policy Insecure FTP
     47 +Finished processing policy Insecure FTP
     48 +Processing policy Insecure FTP with Data Category
     49 +Finished processing policy Insecure FTP with Data Category
     50 +Processing policy Insecure HTTP GET
     51 +Finished processing policy Insecure HTTP GET
     52 +Processing policy Insecure HTTP with Data Category
     53 +Finished processing policy Insecure HTTP with Data Category
     54 +Processing policy Insecure SMTP
     55 +Finished processing policy Insecure SMTP
     56 +Processing policy JWT leaking
     57 +Finished processing policy JWT leaking
     58 +Processing policy Logger leaking
     59 +Finished processing policy Logger leaking
     60 +Processing policy Third-party data category exposure
     61 +Finished processing policy Third-party data category exposure
     62 +Processing policy Session leaking
     63 +Finished processing policy Session leaking
     64 +Processing policy SSL certificate verification disabled
     65 +Finished processing policy SSL certificate verification disabled
     66 +Finished processing policies
    33 67   
    34 68   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-logger_leaking
    skipped 10 lines
    11 11   
    12 12   
    13 13  --
     14 +Processing Detectors
     15 +Finished processing Detectors
     16 +Processing Dataflow
     17 +Finished processing Dataflow
     18 +Processing Policies
     19 +Processing policy Application level encryption missing
     20 +Finished processing policy Application level encryption missing
     21 +Processing policy Cookie leaking
     22 +Finished processing policy Cookie leaking
     23 +Processing policy HTTP GET parameters
     24 +Finished processing policy HTTP GET parameters
     25 +Processing policy Insecure communication
     26 +Finished processing policy Insecure communication
     27 +Processing policy Insecure FTP
     28 +Finished processing policy Insecure FTP
     29 +Processing policy Insecure FTP with Data Category
     30 +Finished processing policy Insecure FTP with Data Category
     31 +Processing policy Insecure HTTP GET
     32 +Finished processing policy Insecure HTTP GET
     33 +Processing policy Insecure HTTP with Data Category
     34 +Finished processing policy Insecure HTTP with Data Category
     35 +Processing policy Insecure SMTP
     36 +Finished processing policy Insecure SMTP
     37 +Processing policy JWT leaking
     38 +Finished processing policy JWT leaking
     39 +Processing policy Logger leaking
     40 +Finished processing policy Logger leaking
     41 +Processing policy Third-party data category exposure
     42 +Finished processing policy Third-party data category exposure
     43 +Processing policy Session leaking
     44 +Finished processing policy Session leaking
     45 +Processing policy SSL certificate verification disabled
     46 +Finished processing policy SSL certificate verification disabled
     47 +Finished processing policies
    14 48   
    15 49   
  • ■ ■ ■ ■ ■ ■
    integration/policies/.snapshots/TestPolicies-sending_data_in_category_to_third_party
    skipped 39 lines
    40 40   
    41 41   
    42 42  --
     43 +Processing Detectors
     44 +Finished processing Detectors
     45 +Processing Dataflow
     46 +Finished processing Dataflow
     47 +Processing Policies
     48 +Processing policy Application level encryption missing
     49 +Finished processing policy Application level encryption missing
     50 +Processing policy Cookie leaking
     51 +Finished processing policy Cookie leaking
     52 +Processing policy HTTP GET parameters
     53 +Finished processing policy HTTP GET parameters
     54 +Processing policy Insecure communication
     55 +Finished processing policy Insecure communication
     56 +Processing policy Insecure FTP
     57 +Finished processing policy Insecure FTP
     58 +Processing policy Insecure FTP with Data Category
     59 +Finished processing policy Insecure FTP with Data Category
     60 +Processing policy Insecure HTTP GET
     61 +Finished processing policy Insecure HTTP GET
     62 +Processing policy Insecure HTTP with Data Category
     63 +Finished processing policy Insecure HTTP with Data Category
     64 +Processing policy Insecure SMTP
     65 +Finished processing policy Insecure SMTP
     66 +Processing policy JWT leaking
     67 +Finished processing policy JWT leaking
     68 +Processing policy Logger leaking
     69 +Finished processing policy Logger leaking
     70 +Processing policy Third-party data category exposure
     71 +Finished processing policy Third-party data category exposure
     72 +Processing policy Session leaking
     73 +Finished processing policy Session leaking
     74 +Processing policy SSL certificate verification disabled
     75 +Finished processing policy SSL certificate verification disabled
     76 +Finished processing policies
    43 77   
    44 78   
  • ■ ■ ■ ■ ■ ■
    pkg/commands/process/balancer/fileignore/fileignore.go
    skipped 2 lines
    3 3  import (
    4 4   "bytes"
    5 5   "io/fs"
     6 + "os"
    6 7   "strings"
    7 8   
    8 9   "github.com/bearer/curio/pkg/commands/process/settings"
    9 10   "github.com/monochromegane/go-gitignore"
     11 + "github.com/rs/zerolog/log"
    10 12  )
    11 13   
    12 14  type FileIgnore struct {
    skipped 16 lines
    29 31   
    30 32   fileInfo, err := d.Info()
    31 33   if err != nil {
     34 + log.Error().Msgf("fileInfo err: %s %s", projectPath, relativePath)
     35 + return true
     36 + }
     37 + 
     38 + symlink, _ := isSymlink(projectPath + relativePath)
     39 + if symlink {
     40 + log.Debug().Msgf("skipping symlink: %s %s", projectPath, relativePath)
    32 41   return true
    33 42   }
    34 43   
    35 44   if fileignore.ignorer.Match(trimmedPath, fileInfo.IsDir()) {
     45 + log.Error().Msgf("file ignore match err: %s %s", projectPath, relativePath)
    36 46   return true
    37 47   }
    38 48   
    39 49   if !fileInfo.IsDir() {
    40 50   if fileInfo.Size() > int64(fileignore.config.Worker.FileSizeMaximum) {
     51 + log.Debug().Msgf("skipping file due to size: %s %s", projectPath, relativePath)
    41 52   return true
    42 53   }
    43 54   }
    44 55   
    45 56   return false
     57 +}
     58 + 
     59 +func isSymlink(path string) (bool, error) {
     60 + fileInfo, err := os.Lstat(path)
     61 + if err != nil {
     62 + return false, err
     63 + }
     64 + 
     65 + return fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink, nil
    46 66  }
    47 67   
    48 68  func ignorerFromStrings(paths []string) gitignore.IgnoreMatcher {
    skipped 8 lines
  • ■ ■ ■ ■ ■ ■
    pkg/commands/process/balancer/filelist/filelist.go
    skipped 35 lines
    36 36   if err != nil {
    37 37   return err
    38 38   }
     39 + 
    39 40   if d.IsDir() {
    40 41   if ignore.Ignore(projectPath, filePath, d) {
    41 42   return filepath.SkipDir
    skipped 36 lines
  • ■ ■ ■ ■ ■ ■
    pkg/commands/process/settings/custom_detector.yml
    skipped 197 lines
    198 198   root_lowercase: true
    199 199   languages:
    200 200   - sql
    201  - processors:
    202  - - query: |
    203  - verified_by = data.bearer.encrypted_verified.verified_by
    204  - encrypted = data.bearer.encrypted_verified.encrypted
    205  - modules:
    206  - - path: processors/encrypted_verified.rego
    207  - name: bearer.encrypted_verified
    208 201   stored: true
    209 202  detect_rails_insecure_smtp:
    210 203   type: "risk"
    skipped 62 lines
  • ■ ■ ■ ■ ■ ■
    pkg/commands/process/settings/processors/encrypted_verified.rego
    skipped 1 lines
    2 2   
    3 3  import future.keywords
    4 4   
    5  - 
    6  -default encrypted := false
    7  - 
    8  - 
    9  -ruby_encrypted[location] {
    10  - some detection in input.all_detections
     5 +ruby_encrypted[detection] {
     6 + detection := input.all_detections[_]
    11 7   detection.detector_type == "detect_encrypted_ruby_class_properties"
    12 8   detection.value.classification.decision.state == "valid"
    13  - location = detection
    14 9  }
    15 10   
    16  -encrypted = true {
    17  - some detection in ruby_encrypted
    18  - detection.value.object_name == input.target.value.object_name
    19  - detection.value.field_name == input.target.value.field_name
    20  - input.target.value.field_name != ""
    21  - input.target.value.object_name != ""
     11 +encrypted[detection] {
     12 + detection := input.target_detections[_]
     13 + detection.value.field_name != ""
     14 + detection.value.object_name != ""
     15 + 
     16 + some encrypted_detection in ruby_encrypted
     17 + detection.value.object_name == encrypted_detection.value.object_name
     18 + detection.value.field_name == encrypted_detection.value.field_name
    22 19  }
    23 20   
    24  -verified_by[verification] {
    25  - some detection in ruby_encrypted
    26  - detection.value.object_name == input.target.value.object_name
    27  - detection.value.field_name == input.target.value.field_name
     21 +verified_by[[detection, verifications]] {
     22 + detection := input.target_detections[_]
     23 + detection.value.field_name != ""
     24 + detection.value.object_name != ""
    28 25   
    29  - verification = {
    30  - "detector": "detect_encrypted_ruby_class_properties",
    31  - "filename": detection.source.filename,
    32  - "line_number": detection.source.line_number
    33  - }
     26 + verifications := [verification |
     27 + encrypted_detection := ruby_encrypted[_]
     28 + detection.value.object_name == encrypted_detection.value.object_name
     29 + detection.value.field_name == encrypted_detection.value.field_name
     30 + verification := {
     31 + "detector": "detect_encrypted_ruby_class_properties",
     32 + "filename": encrypted_detection.source.filename,
     33 + "line_number": encrypted_detection.source.line_number
     34 + }]
     35 + 
     36 + count(verifications) != 0
    34 37  }
     38 + 
  • ■ ■ ■ ■ ■ ■
    pkg/commands/process/settings/settings.go
    skipped 69 lines
    70 70   Type string `mapstructure:"type" json:"type" yaml:"type"`
    71 71   Languages []string `mapstructure:"languages" json:"languages" yaml:"languages"`
    72 72   ParamParenting bool `mapstructure:"param_parenting" json:"param_parenting" yaml:"param_parenting"`
    73  - Processors []Processor `mapstructure:"processors" json:"processors" yaml:"processors"`
    74 73   Patterns []RulePattern `mapstructure:"patterns" json:"patterns" yaml:"patterns"`
    75 74   
    76 75   RootSingularize bool `mapstructure:"root_singularize" yaml:"root_singularize" `
    skipped 40 lines
    117 116   }
    118 117   } else {
    119 118   rules = DefaultCustomDetector()
    120  - }
    121  - 
    122  - for _, customDetector := range rules {
    123  - for _, processor := range customDetector.Processors {
    124  - for _, module := range processor.Modules {
    125  - if module.Path != "" {
    126  - content, err := processorsFs.ReadFile(module.Path)
    127  - if err != nil {
    128  - return Config{}, err
    129  - }
    130  - module.Content = string(content)
    131  - module.Path = ""
    132  - }
    133  - }
    134  - }
    135 119   }
    136 120   
    137 121   var policies map[string]*Policy
    skipped 76 lines
    214 198   return policies
    215 199  }
    216 200   
     201 +func EncryptedVerifiedRegoModuleText() (string, error) {
     202 + data, err := processorsFs.ReadFile("processors/encrypted_verified.rego")
     203 + if err != nil {
     204 + return "", err
     205 + }
     206 + 
     207 + return string(data), nil
     208 +}
     209 + 
  • ■ ■ ■ ■ ■ ■
    pkg/flag/worker_flags.go
    skipped 43 lines
    44 44   FileSizeMaximumFlag = Flag{
    45 45   Name: "file-size-max",
    46 46   ConfigName: "worker.file-size-max",
    47  - Value: 100 * 1000, // 100 KB
     47 + Value: 2 * 1000 * 1000, // 2MB
    48 48   Usage: "Ignore files larger than the specified value.",
    49 49   }
    50 50   FilesToBatchFlag = Flag{
    skipped 95 lines
  • ■ ■ ■ ■ ■ ■
    pkg/report/output/dataflow/dataflow.go
    skipped 6 lines
    7 7   "strings"
    8 8   
    9 9   "github.com/bearer/curio/pkg/commands/process/settings"
    10  - reportdetectors "github.com/bearer/curio/pkg/report/detectors"
    11 10   "github.com/bearer/curio/pkg/report/customdetectors"
    12 11   "github.com/bearer/curio/pkg/report/detections"
     12 + reportdetectors "github.com/bearer/curio/pkg/report/detectors"
    13 13   "github.com/bearer/curio/pkg/report/output/dataflow/components"
    14 14   "github.com/bearer/curio/pkg/report/output/dataflow/datatypes"
    15 15   "github.com/bearer/curio/pkg/report/output/dataflow/detectiondecoder"
    skipped 3 lines
    19 19  )
    20 20   
    21 21  type DataFlow struct {
    22  - Datatypes []types.Datatype `json:"data_types,omitempty" yaml:"data_types,omitempty"`
    23  - Risks []interface{} `json:"risks,omitempty" yaml:"risks,omitempty"`
    24  - Components []types.Component `json:"components" yaml:"components"`
     22 + Datatypes []types.Datatype `json:"data_types,omitempty" yaml:"data_types,omitempty"`
     23 + Risks []interface{} `json:"risks,omitempty" yaml:"risks,omitempty"`
     24 + Components []types.Component `json:"components" yaml:"components"`
    25 25  }
    26 26   
    27 27  var allowedDetections []detections.DetectionType = []detections.DetectionType{detections.TypeSchemaClassified, detections.TypeCustomClassified, detections.TypeDependencyClassified, detections.TypeInterfaceClassified, detections.TypeFrameworkClassified, detections.TypeCustomRisk}
    skipped 2 lines
    30 30   dataTypesHolder := datatypes.New(config, isInternal)
    31 31   risksHolder := risks.New(config, isInternal)
    32 32   componentsHolder := components.New(isInternal)
     33 + 
     34 + extras, err := datatypes.NewExtras(input)
     35 + if err != nil {
     36 + return nil, err
     37 + }
     38 + railsExtras, err := datatypes.NewRailsExtras(input)
     39 + if err != nil {
     40 + return nil, err
     41 + }
    33 42   
    34 43   for _, detection := range input {
    35 44   detectionMap, ok := detection.(map[string]interface{})
    skipped 36 lines
    72 81   
    73 82   switch detectionType {
    74 83   case detections.TypeSchemaClassified:
     84 + var detectionExtras *datatypes.ExtraFields
    75 85   if castDetection.DetectorType == reportdetectors.DetectorSchemaRb {
    76  - extras, err := datatypes.GetRailsExtras(input, detection)
    77  - if err != nil {
    78  - return nil, err
    79  - }
     86 + detectionExtras = railsExtras.Get(detection)
     87 + }
    80 88   
    81  - err = dataTypesHolder.AddSchema(castDetection, extras)
    82  - if err != nil {
    83  - return nil, err
    84  - }
    85  - } else {
    86  - err = dataTypesHolder.AddSchema(castDetection, nil)
    87  - if err != nil {
    88  - return nil, err
    89  - }
     89 + err = dataTypesHolder.AddSchema(castDetection, detectionExtras)
     90 + if err != nil {
     91 + return nil, err
    90 92   }
    91 93   case detections.TypeCustomRisk:
    92 94   risksHolder.AddRiskPresence(castDetection)
    skipped 13 lines
    106 108   return nil, err
    107 109   }
    108 110   case customdetectors.TypeDatatype:
    109  - extras, err := datatypes.GetExtras(customDetector, input, detection)
    110  - if err != nil {
    111  - return nil, err
     111 + var detectionExtras *datatypes.ExtraFields
     112 + if castDetection.DetectorType == "detect_sql_create_public_table" {
     113 + detectionExtras = extras.Get(detection)
    112 114   }
    113 115   
    114  - err = dataTypesHolder.AddSchema(castDetection, extras)
     116 + err = dataTypesHolder.AddSchema(castDetection, detectionExtras)
    115 117   if err != nil {
    116 118   return nil, err
    117 119   }
    skipped 62 lines
  • ■ ■ ■ ■ ■ ■
    pkg/report/output/dataflow/datatypes/datatypes.go
    skipped 10 lines
    11 11   "github.com/bearer/curio/pkg/report/detections"
    12 12   "github.com/bearer/curio/pkg/util/classify"
    13 13   "github.com/bearer/curio/pkg/util/maputil"
     14 + "github.com/bearer/curio/pkg/util/output"
    14 15  )
    15 16   
    16 17  type Holder struct {
    skipped 36 lines
    53 54   }
    54 55  }
    55 56   
    56  -func (holder *Holder) AddSchema(detection detections.Detection, extras *extraFields) error {
     57 +func (holder *Holder) AddSchema(detection detections.Detection, extras *ExtraFields) error {
    57 58   schema, err := detectiondecoder.GetSchema(detection)
    58 59   if err != nil {
    59 60   return err
    skipped 12 lines
    72 73  }
    73 74   
    74 75  // addDatatype adds datatype to hash list and at the same time blocks duplicates
    75  -func (holder *Holder) addDatatype(classification *db.DataType, detectorName string, fileName string, lineNumber int, extras *extraFields, parent *schema.Parent) {
     76 +func (holder *Holder) addDatatype(classification *db.DataType, detectorName string, fileName string, lineNumber int, extras *ExtraFields, parent *schema.Parent) {
    76 77   // create datatype entry if it doesn't exist
    77 78   if _, exists := holder.datatypes[classification.Name]; !exists {
    78 79   datatype := datatypeHolder{
    skipped 55 lines
    134 135  }
    135 136   
    136 137  func (holder *Holder) ToDataFlow() []types.Datatype {
     138 + output.StdErrLogger().Msgf("Processing Dataflow")
    137 139   data := make([]types.Datatype, 0)
    138 140   
    139 141   datatypes := maputil.ToSortedSlice(holder.datatypes)
    skipped 31 lines
    171 173   
    172 174   data = append(data, constructedDatatype)
    173 175   }
     176 + output.StdErrLogger().Msgf("Finished processing Dataflow")
    174 177   
    175 178   return data
    176 179  }
    skipped 1 lines
  • ■ ■ ■ ■ ■ ■
    pkg/report/output/dataflow/datatypes/get_extras.go
    skipped 1 lines
    2 2   
    3 3  import (
    4 4   "encoding/json"
     5 + "errors"
     6 + "fmt"
    5 7   
    6 8   "github.com/bearer/curio/pkg/commands/process/settings"
     9 + "github.com/bearer/curio/pkg/report/detections"
     10 + "github.com/bearer/curio/pkg/report/detectors"
    7 11   "github.com/bearer/curio/pkg/report/output/dataflow/types"
    8 12   regohelper "github.com/bearer/curio/pkg/util/rego"
    9 13   "github.com/open-policy-agent/opa/rego"
    10 14  )
    11 15   
    12 16  type processorInput struct {
    13  - AllDetections []interface{} `json:"all_detections"`
    14  - Target interface{} `json:"target"`
     17 + AllDetections []interface{} `json:"all_detections"`
     18 + TargetDetections []interface{} `json:"target_detections"`
    15 19  }
    16 20   
    17  -type extraFields struct {
     21 +type ExtraFields struct {
    18 22   encrypted *bool
    19 23   verifiedBy []types.DatatypeVerifiedBy
    20 24  }
    21 25   
    22  -func GetRailsExtras(input []interface{}, detection interface{}) (*extraFields, error) {
    23  - extras := &extraFields{}
     26 +func getRailsTargetDetections(allDetections []interface{}) ([]interface{}, error) {
     27 + var result []interface{}
    24 28   
    25  - processorContent := `
    26  -package bearer.rails_encrypted_verified
     29 + for _, detection := range allDetections {
     30 + detectionMap, ok := detection.(map[string]interface{})
     31 + if !ok {
     32 + return nil, fmt.Errorf("found detection in report which is not object")
     33 + }
    27 34   
    28  -import future.keywords
     35 + detectionType, ok := detectionMap["type"].(string)
     36 + if !ok {
     37 + continue
     38 + }
    29 39   
    30  -default encrypted := false
     40 + if detections.DetectionType(detectionType) != detections.TypeSchemaClassified {
     41 + continue
     42 + }
    31 43   
    32  -ruby_encrypted[location] {
    33  - some detection in input.all_detections
    34  - detection.detector_type == "detect_encrypted_ruby_class_properties"
    35  - detection.value.classification.decision.state == "valid"
    36  - location = detection
     44 + detectorType, ok := detectionMap["detector_type"].(string)
     45 + if !ok {
     46 + continue
     47 + }
     48 + 
     49 + if detectors.Type(detectorType) != detectors.DetectorSchemaRb {
     50 + continue
     51 + }
     52 + 
     53 + result = append(result, detection)
     54 + }
     55 + 
     56 + return result, nil
    37 57  }
    38 58   
    39  -encrypted = true {
    40  - some detection in ruby_encrypted
    41  - detection.value.transformed_object_name == input.target.value.transformed_object_name
    42  - detection.value.field_name == input.target.value.field_name
    43  - input.target.value.field_name != ""
    44  - input.target.value.object_name != ""
    45  -}
     59 +func getEncryptedField(result rego.Vars, detection interface{}) (bool, error) {
     60 + rawEncryptedFields, ok := result["encrypted"]
     61 + if !ok {
     62 + return false, errors.New("no 'encrypted' value in output")
     63 + }
     64 + 
     65 + encryptedFields, ok := rawEncryptedFields.([]interface{})
     66 + if !ok {
     67 + return false, errors.New("invalid type for 'encrypted' value")
     68 + }
     69 + 
     70 + detectionMap := detection.(map[string]interface{})
     71 + detectionID := detectionMap["id"].(string)
     72 + 
     73 + for _, rawResultDetection := range encryptedFields {
     74 + resultDetection, ok := rawResultDetection.(map[string]interface{})
     75 + if !ok {
     76 + return false, errors.New("invalid type for 'encrypted' detection")
     77 + }
     78 + 
     79 + rawResultDetectionID, ok := resultDetection["id"]
     80 + if !ok {
     81 + return false, errors.New("missing id for 'encrypted' detection")
     82 + }
    46 83   
    47  -verified_by[verification] {
    48  - some detection in ruby_encrypted
    49  - detection.value.transformed_object_name == input.target.value.transformed_object_name
    50  - detection.value.field_name == input.target.value.field_name
     84 + resultDetectionID, ok := rawResultDetectionID.(string)
     85 + if !ok {
     86 + return false, errors.New("invalid type for 'encrypted' detection id")
     87 + }
    51 88   
    52  - verification = {
    53  - "detector": "detect_encrypted_ruby_class_properties",
    54  - "filename": detection.source.filename,
    55  - "line_number": detection.source.line_number
     89 + if resultDetectionID == detectionID {
     90 + return true, nil
    56 91   }
     92 + }
     93 + 
     94 + return false, nil
    57 95  }
    58  -`
    59 96   
    60  - query := `
    61  -verified_by = data.bearer.rails_encrypted_verified.verified_by
    62  -encrypted = data.bearer.rails_encrypted_verified.encrypted
    63  -`
     97 +func getVerifiedBy(result rego.Vars, detection interface{}) ([]types.DatatypeVerifiedBy, error) {
     98 + rawVerifiedBy, ok := result["verified_by"]
     99 + if !ok {
     100 + return nil, errors.New("no 'verified_by' value in output")
     101 + }
    64 102   
    65  - module := regohelper.Module{
    66  - Name: "bearer.rails_encrypted_verified",
    67  - Content: processorContent,
     103 + verifiedBy, ok := rawVerifiedBy.([]interface{})
     104 + if !ok {
     105 + return nil, errors.New("invalid type for 'verified_by' value")
    68 106   }
    69 107   
    70  - result, err := regohelper.RunQuery(query, processorInput{
    71  - AllDetections: input,
    72  - Target: detection,
    73  - }, []regohelper.Module{module})
    74  - if err != nil {
    75  - return nil, err
    76  - }
     108 + detectionMap := detection.(map[string]interface{})
     109 + detectionID := detectionMap["id"].(string)
    77 110   
    78  - encrypted := getEncryptedField(result)
     111 + for _, rawItem := range verifiedBy {
     112 + item, ok := rawItem.([]interface{})
     113 + if !ok {
     114 + return nil, errors.New("invalid type for 'verified_by' item")
     115 + }
    79 116   
    80  - if encrypted {
    81  - extras.encrypted = &encrypted
     117 + if len(item) != 2 {
     118 + return nil, errors.New("invalid length for 'verified_by' item")
     119 + }
    82 120   
    83  - verified, err := getVerifiedBy(result)
    84  - if err != nil {
    85  - return nil, err
     121 + rawItemDetection := item[0]
     122 + rawItemVerifiedBy := item[1]
     123 + 
     124 + itemDetection, ok := rawItemDetection.(map[string]interface{})
     125 + if !ok {
     126 + return nil, errors.New("invalid type for 'verified_by' item detection")
    86 127   }
    87 128   
    88  - if verified != nil {
    89  - extras.verifiedBy = append(extras.verifiedBy, verified...)
     129 + rawItemDetectionID, ok := itemDetection["id"]
     130 + if !ok {
     131 + return nil, errors.New("missing id for 'verified_by' item detection")
    90 132   }
     133 + 
     134 + itemDetectionID, ok := rawItemDetectionID.(string)
     135 + if !ok {
     136 + return nil, errors.New("invalid type for 'verified_by' item detection id")
     137 + }
     138 + 
     139 + if itemDetectionID != detectionID {
     140 + continue
     141 + }
     142 + 
     143 + var verifiedBy []types.DatatypeVerifiedBy
     144 + bytes, err := json.Marshal(rawItemVerifiedBy)
     145 + if err != nil {
     146 + return nil, fmt.Errorf("failed to serialize 'verified_by' item: %s", err)
     147 + }
     148 + err = json.Unmarshal(bytes, &verifiedBy)
     149 + if err != nil {
     150 + return nil, fmt.Errorf("invalid format for 'verified_by' item: %s", err)
     151 + }
     152 + 
     153 + return verifiedBy, nil
    91 154   }
    92 155   
    93  - return extras, nil
     156 + return nil, nil
     157 +}
     158 + 
     159 +type extrasObj struct {
     160 + data map[string]*ExtraFields
    94 161  }
    95 162   
    96  -func GetExtras(customDetector settings.Rule, input []interface{}, detection interface{}) (*extraFields, error) {
    97  - extras := &extraFields{}
     163 +func NewRailsExtras(detections []interface{}) (*extrasObj, error) {
     164 + return newExtrasObj(detections, getRailsTargetDetections)
     165 +}
     166 + 
     167 +func NewExtras(detections []interface{}) (*extrasObj, error) {
     168 + return newExtrasObj(detections, getTargetDetections)
     169 +}
     170 + 
     171 +func newExtrasObj(
     172 + detections []interface{},
     173 + targetDetectionsFunc func(detections []interface{}) ([]interface{}, error),
     174 +) (*extrasObj, error) {
     175 + targetDetections, err := targetDetectionsFunc(detections)
     176 + if err != nil {
     177 + return nil, err
     178 + }
     179 + 
     180 + module, err := settings.EncryptedVerifiedRegoModuleText()
     181 + if err != nil {
     182 + return nil, err
     183 + }
     184 + 
     185 + data, err := runExtrasQuery(
     186 + `
     187 + verified_by = data.bearer.encrypted_verified.verified_by
     188 + encrypted = data.bearer.encrypted_verified.encrypted
     189 + `,
     190 + []regohelper.Module{{
     191 + Name: "bearer.encrypted_verified",
     192 + Content: module,
     193 + }},
     194 + detections,
     195 + targetDetections,
     196 + )
     197 + if err != nil {
     198 + return nil, err
     199 + }
     200 + 
     201 + return &extrasObj{data: data}, nil
     202 +}
     203 + 
     204 +func runExtrasQuery(
     205 + query string,
     206 + modules []regohelper.Module,
     207 + detections, targetDetections []interface{},
     208 +) (map[string]*ExtraFields, error) {
     209 + data := make(map[string]*ExtraFields)
     210 + 
     211 + result, err := regohelper.RunQuery(query, processorInput{
     212 + AllDetections: detections,
     213 + TargetDetections: targetDetections,
     214 + }, modules)
     215 + if err != nil {
     216 + return nil, err
     217 + }
    98 218   
    99  - for _, processor := range customDetector.Processors {
    100  - result, err := regohelper.RunQuery(processor.Query, processorInput{
    101  - AllDetections: input,
    102  - Target: detection,
    103  - }, processor.Modules.ToRegoModules())
     219 + for _, detection := range targetDetections {
     220 + extras := &ExtraFields{}
     221 + encrypted, err := getEncryptedField(result, detection)
    104 222   if err != nil {
    105 223   return nil, err
    106 224   }
    107  - 
    108  - encrypted := getEncryptedField(result)
    109 225   
    110 226   if encrypted {
    111 227   extras.encrypted = &encrypted
    112 228   
    113  - verified, err := getVerifiedBy(result)
     229 + verified, err := getVerifiedBy(result, detection)
    114 230   if err != nil {
    115 231   return nil, err
    116 232   }
    skipped 2 lines
    119 235   extras.verifiedBy = append(extras.verifiedBy, verified...)
    120 236   }
    121 237   }
     238 + 
     239 + detectionMap := detection.(map[string]interface{})
     240 + detectionID := detectionMap["id"].(string)
     241 + data[detectionID] = extras
    122 242   }
    123 243   
    124  - return extras, nil
     244 + return data, nil
    125 245  }
    126 246   
    127  -func getEncryptedField(result rego.Vars) bool {
    128  - encryptedField, hasEncryptedField := result["encrypted"]
     247 +func getTargetDetections(allDetections []interface{}) ([]interface{}, error) {
     248 + var result []interface{}
     249 + 
     250 + for _, detection := range allDetections {
     251 + detectionMap, ok := detection.(map[string]interface{})
     252 + if !ok {
     253 + return nil, fmt.Errorf("found detection in report which is not object")
     254 + }
    129 255   
    130  - if hasEncryptedField {
    131  - encrypted, ok := encryptedField.(bool)
     256 + detectionTypeS, ok := detectionMap["type"].(string)
     257 + if !ok {
     258 + continue
     259 + }
    132 260   
    133  - if ok && encrypted {
    134  - return true
     261 + detectionType := detections.DetectionType(detectionTypeS)
     262 + if detectionType != detections.TypeCustomClassified {
     263 + continue
    135 264   }
     265 + 
     266 + result = append(result, detection)
    136 267   }
    137 268   
    138  - return false
     269 + return result, nil
    139 270  }
    140 271   
    141  -func getVerifiedBy(result rego.Vars) ([]types.DatatypeVerifiedBy, error) {
    142  - verifiedByField, hasverifiedByField := result["verified_by"]
    143  - 
    144  - if hasverifiedByField {
    145  - var verifiedBy []types.DatatypeVerifiedBy
    146  - bytes, err := json.Marshal(verifiedByField)
    147  - if err != nil {
    148  - return nil, err
    149  - }
    150  - err = json.Unmarshal(bytes, &verifiedBy)
    151  - if err != nil {
    152  - return nil, err
    153  - }
     272 +func (extras *extrasObj) Get(detection interface{}) *ExtraFields {
     273 + detectionMap := detection.(map[string]interface{})
     274 + detectionID := detectionMap["id"].(string)
    154 275   
    155  - return verifiedBy, nil
    156  - }
    157  - 
    158  - return nil, nil
     276 + return extras.data[detectionID]
    159 277  }
    160 278   
  • ■ ■ ■ ■ ■ ■
    pkg/report/output/dataflow/risks/risks_test.go
    skipped 29 lines
    30 30   Name string
    31 31   Config settings.Config
    32 32   FileContent string
    33  - Want []interface {}
     33 + Want []interface{}
    34 34   }{
    35 35   {
    36 36   Name: "single detection",
    37 37   Config: config,
    38  - FileContent: `{"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
    39  - Want: []interface {}{
     38 + FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
     39 + Want: []interface{}{
    40 40   types.RiskDetector{
    41 41   DetectorID: "detect_ruby_logger",
    42 42   DataTypes: []types.RiskDatatype{
    skipped 11 lines
    54 54   {
    55 55   Name: "single detection - no classification",
    56 56   Config: config,
    57  - FileContent: `{"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name"}}`,
    58  - Want: []interface {}{},
     57 + FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name"}}`,
     58 + Want: []interface{}{},
    59 59   },
    60 60   {
    61 61   Name: "single detection - duplicates",
    62 62   Config: config,
    63  - FileContent: `{"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
    64  - {"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
    65  - Want: []interface {}{
     63 + FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
     64 + {"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
     65 + Want: []interface{}{
    66 66   types.RiskDetector{
    67 67   DetectorID: "detect_ruby_logger",
    68 68   DataTypes: []types.RiskDatatype{
    skipped 11 lines
    80 80   {
    81 81   Name: "single detection - stored",
    82 82   Config: config,
    83  - FileContent: `{"type": "custom_classified", "detector_type":"ruby_leak", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
    84  - Want: []interface {}{
     83 + FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"ruby_leak", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
     84 + Want: []interface{}{
    85 85   types.RiskDetector{
    86 86   DetectorID: "ruby_leak",
    87 87   DataTypes: []types.RiskDatatype{
    skipped 11 lines
    99 99   {
    100 100   Name: "single detection - multiple occurences - deterministic output",
    101 101   Config: config,
    102  - FileContent: `{"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
    103  - {"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 2}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
    104  - Want: []interface {}{
     102 + FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
     103 + {"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 2}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
     104 + Want: []interface{}{
    105 105   types.RiskDetector{
    106 106   DetectorID: "detect_ruby_logger",
    107 107   DataTypes: []types.RiskDatatype{
    skipped 12 lines
    120 120   {
    121 121   Name: "multiple detections - same detector - deterministic output",
    122 122   Config: config,
    123  - FileContent: `{"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
    124  - {"type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./address.rb", "line_number": 2}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Physical Address", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
    125  - Want: []interface {}{
     123 + FileContent: `{"id": "1", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./users.rb", "line_number": 25}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Username", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}
     124 + {"id": "2", "type": "custom_classified", "detector_type":"detect_ruby_logger", "source": {"filename": "./address.rb", "line_number": 2}, "value": {"field_name": "User_name", "classification": {"data_type": {"name": "Physical Address", "uuid": "123", "category_uuid": "456"} ,"decision":{"state": "valid"}}}}`,
     125 + Want: []interface{}{
    126 126   types.RiskDetector{
    127 127   DetectorID: "detect_ruby_logger",
    128 128   DataTypes: []types.RiskDatatype{
    skipped 54 lines
  • ■ ■ ■ ■ ■ ■
    pkg/report/output/detectors/detectors.go
    skipped 4 lines
    5 5   "os"
    6 6   
    7 7   "github.com/bearer/curio/pkg/types"
     8 + "github.com/bearer/curio/pkg/util/output"
    8 9   "github.com/rs/zerolog/log"
    9 10   "github.com/wlredeye/jsonlines"
    10 11  )
    11 12   
    12 13  func GetOutput(report types.Report) ([]interface{}, error) {
     14 + output.StdErrLogger().Msgf("Processing Detectors")
    13 15   var detections []interface{}
    14 16   f, err := os.Open(report.Path)
    15 17   if err != nil {
    skipped 6 lines
    22 24   }
    23 25   log.Debug().Msgf("got %d detections", len(detections))
    24 26   
     27 + output.StdErrLogger().Msgf("Finished processing Detectors")
    25 28   return detections, nil
    26 29  }
    27 30   
  • ■ ■ ■ ■ ■ ■
    pkg/report/output/output.go
    skipped 7 lines
    8 8   "github.com/bearer/curio/pkg/commands/process/settings"
    9 9   "github.com/bearer/curio/pkg/flag"
    10 10   "github.com/bearer/curio/pkg/report/output/dataflow"
     11 + "github.com/google/uuid"
    11 12   
    12 13   "github.com/bearer/curio/pkg/report/output/detectors"
    13 14   "github.com/bearer/curio/pkg/report/output/policies"
    skipped 105 lines
    119 120   reportedDetections, err := detectors.GetOutput(report)
    120 121   if err != nil {
    121 122   return nil, err
     123 + }
     124 + 
     125 + for _, detection := range reportedDetections {
     126 + detection.(map[string]interface{})["id"] = uuid.NewString()
    122 127   }
    123 128   
    124 129   return dataflow.GetOutput(reportedDetections, config, isInternal)
    skipped 2 lines
  • ■ ■ ■ ■ ■ ■ ■
    pkg/report/output/policies/policies.go
    skipped 2 lines
    3 3  import (
    4 4   "encoding/json"
    5 5   "fmt"
     6 + "sort"
    6 7   "strings"
    7 8   
    8 9   "github.com/bearer/curio/pkg/classification/db"
    9 10   "github.com/bearer/curio/pkg/commands/process/settings"
    10 11   "github.com/bearer/curio/pkg/util/file"
     12 + "github.com/bearer/curio/pkg/util/output"
    11 13   "github.com/bearer/curio/pkg/util/rego"
    12 14   "github.com/fatih/color"
    13 15   "golang.org/x/exp/maps"
    skipped 37 lines
    51 53  }
    52 54   
    53 55  func GetOutput(dataflow *dataflow.DataFlow, config settings.Config) (map[string][]PolicyResult, error) {
     56 + output.StdErrLogger().Msgf("Processing Policies")
    54 57   // policy results grouped by severity (critical, high, ...)
    55 58   result := make(map[string][]PolicyResult)
    56 59   
    57  - for _, policy := range config.Policies {
     60 + // Ensure a deterministic order
     61 + keys := maps.Keys(config.Policies)
     62 + sort.Strings(keys)
     63 + 
     64 + for _, key := range keys {
     65 + policy := config.Policies[key]
     66 + output.StdErrLogger().Msgf("Processing policy %s", policy.Name)
     67 + 
    58 68   // Create a prepared query that can be evaluated.
    59 69   rs, err := rego.RunQuery(policy.Query,
    60 70   PolicyInput{
    skipped 33 lines
    94 104   result[policyOutput.Severity] = append(result[policyOutput.Severity], policyResult)
    95 105   }
    96 106   }
     107 + 
     108 + output.StdErrLogger().Msgf("Finished processing policy %s", policy.Name)
    97 109   }
    98 110   
     111 + output.StdErrLogger().Msgf("Finished processing policies")
    99 112   return result, nil
    100 113  }
    101 114   
    skipped 144 lines
  • ■ ■ ■ ■ ■ ■
    pkg/report/writer/detectors.go
    skipped 27 lines
    28 28  )
    29 29   
    30 30  type StoredSchema struct {
    31  - Value schema.Schema
    32  - Source *source.Source
    33  - Parent *parser.Node
     31 + Value schema.Schema
     32 + Source *source.Source
     33 + Parent *parser.Node
    34 34  }
    35 35   
    36 36  type StoredSchemaNodes = map[*parser.Node]*StoredSchema
    skipped 73 lines
    110 110   report.StoredSchemas = &SchemaGroup{
    111 111   Node: node,
    112 112   ParentSchema: StoredSchema{
    113  - Value: schema,
     113 + Value: schema,
    114 114   Source: source,
    115 115   Parent: parent,
    116 116   },
    117 117   DetectorType: detectorType,
    118  - Schemas: make(StoredSchemaNodes),
     118 + Schemas: make(StoredSchemaNodes),
    119 119   }
    120 120  }
    121 121   
    skipped 20 lines
    142 142   
    143 143   childName := schema.FieldName
    144 144   childDataTypes[childName] = &datatype.DataType{
    145  - Node: node,
    146  - Name: childName,
    147  - Type: schema.SimpleFieldType,
    148  - TextType: schema.FieldType,
     145 + Node: node,
     146 + Name: childName,
     147 + Type: schema.SimpleFieldType,
     148 + TextType: schema.FieldType,
    149 149   Properties: map[string]datatype.DataTypable{},
    150  - UUID: schema.FieldUUID,
     150 + UUID: schema.FieldUUID,
    151 151   }
    152 152   }
    153 153   
    154 154   // Build parent data type
    155 155   parentDataType := &datatype.DataType{
    156  - Node: report.StoredSchemas.Node,
    157  - Name: report.StoredSchemas.ParentSchema.Value.ObjectName,
    158  - Type: "",
    159  - TextType: "",
     156 + Node: report.StoredSchemas.Node,
     157 + Name: report.StoredSchemas.ParentSchema.Value.ObjectName,
     158 + Type: "",
     159 + TextType: "",
    160 160   Properties: childDataTypes,
    161  - UUID: report.StoredSchemas.ParentSchema.Value.ObjectUUID,
     161 + UUID: report.StoredSchemas.ParentSchema.Value.ObjectUUID,
    162 162   }
    163 163   
    164 164   classifiedDatatypes := make(map[parser.NodeID]*classificationschema.ClassifiedDatatype, 0)
    skipped 113 lines
  • ■ ■ ■ ■ ■ ■
    pkg/util/rego/rego.go
    skipped 32 lines
    33 33   }
    34 34   
    35 35   if len(rs) != 1 {
    36  - return nil, fmt.Errorf("expected single result from query got %d results %#v", len(rs), rs)
     36 + return nil, fmt.Errorf("expected single result from query got %d results %#v:\n%s", len(rs), rs, query)
    37 37   }
    38 38   
    39 39   return rs[0].Bindings, nil
    skipped 2 lines
Please wait...
Page is in error, reload to recover