STRLCPY/Osmedeus

Update Next Generation
j3ssie committed 2 years ago

8127be3d

Showing first 30 files as there are too many

■ ■ ■ ■ ■ ■

.gitignore

1 + .DS_STORE
2 + .idea
3 + out*
4 + osmedeus
5 + dist/*
6 +

All occurrences

■ ■ ■ ■ ■ ■

Makefile

1	+	TARGET ?= osmedeus
2	+	GO ?= go
3	+	GOFLAGS ?=
4	+
5	+	build:
6	+	go install
7	+	rm -rf ./dist/* ~/myGit/premium-osmedeus-base/dist/* ~/org-osmedeus/osmedeus-base/dist/*
8	+	# GOOS=darwin GOARCH=amd64 $(GO) build $(GOFLAGS) -o dist/osmedeus
9	+	# zip -j dist/osmedeus-darwin.zip dist/osmedeus
10	+	# rm -rf ./dist/osmedeus
11	+	# for linux build on mac
12	+	GOOS=linux GOARCH=amd64 CC="/usr/local/bin/x86_64-linux-musl-gcc" CGO_ENABLED=1 go build -ldflags "-linkmode external -extldflags -static" -o dist/osmedeus
13	+	zip -j dist/osmedeus-linux.zip dist/osmedeus
14	+	rm -rf ./dist/osmedeus
15	+	cp dist/* ~/myGit/premium-osmedeus-base/dist/
16	+	cp dist/* ~/org-osmedeus/osmedeus-base/dist/
17	+	run:
18	+	$(GO) $(GOFLAGS) run *.go
19	+
20	+	fmt:
21	+	$(GO) $(GOFLAGS) fmt ./...; \
22	+	echo "Done."
23	+
24	+	test:
25	+	$(GO) $(GOFLAGS) test ./... -v%

■ ■ ■ ■ ■ ■

README.md

1	+	# Osmedeus Core Engine
2	+
3	+	<p align="center">
4	+	<img alt="Osmedeus" src="https://raw.githubusercontent.com/osmedeus/assets/main/logo-transparent.png" height="140" />
5	+	<br />
6	+	<strong>Osmedeus - A Workflow Engine for Offensive Security</strong>
7	+	<p align="center">
8	+	<a href="https://docs.osmedeus.org/"><img src="https://img.shields.io/badge/Documentation-0078D4?style=for-the-badge&logo=Google-Chrome&logoColor=39ff14&labelColor=black&color=black"></a>
9	+	<a href="https://docs.osmedeus.org/donation/"><img src="https://img.shields.io/badge/Sponsors-0078D4?style=for-the-badge&logo=GitHub-Sponsors&logoColor=39ff14&labelColor=black&color=black"></a>
10	+	<a href="https://twitter.com/OsmedeusEngine"><img src="https://img.shields.io/badge/%40OsmedeusEngine-0078D4?style=for-the-badge&logo=Twitter&logoColor=39ff14&labelColor=black&color=black"></a>
11	+	<a href="https://discord.gg/gy4SWhpaPU"><img src="https://img.shields.io/badge/Discord%20Server-0078D4?style=for-the-badge&logo=Discord&logoColor=39ff14&labelColor=black&color=black"></a>
12	+	</p>
13	+	</p>
14	+
15	+	***
16	+
17	+	## Installation
18	+
19	+	> NOTE that you need some essential tools like `curl, wget, git, zip` and login as root to start
20	+
21	+	```shell
22	+	bash -c "$(curl -fsSL https://raw.githubusercontent.com/osmedeus/osmedeus-base/master/install.sh)"
23	+	```
24	+
25	+	## Usage
26	+
27	+	```shell
28	+	# Practical Usage:
29	+	osmedeus scan -f [flowName] -t [target]
30	+	osmedeus scan -f [flowName] -T [targetsFile]
31	+	osmedeus scan -f /path/to/flow.yaml -t [target]
32	+	osmedeus scan -m /path/to/module.yaml -t [target] --params 'port=9200'
33	+	osmedeus scan -m /path/to/module.yaml -t [target] -l /tmp/log.log
34	+	cat targets \| osmedeus scan -f sample
35	+
36	+	# Example Commands:
37	+	osmedeus scan -t target.com
38	+	osmedeus scan -T list_of_targets.txt -W custom_workspaces
39	+	osmedeus scan -t target.com -w workspace_name --debug
40	+	osmedeus scan -f single -t www.sample.com
41	+	osmedeus scan -f ovuln-T list_of_target.txt
42	+	osmedeus scan -m ~/osmedeus-base/workflow/test/dirbscan.yaml -t list_of_urls.txt
43	+	osmedeus health
44	+	ls ~/.osmedeus/storages/summary/ \| osmedeus scan -m ~/osmedeus-base/workflow/test/dirbscan.yaml
45	+	ls ~/.osmedeus/storages/summary/ \| osmedeus scan -m ~/osmedeus-base/workflow/test/busting.yaml -D
46	+
47	+	# Start Web UI at https://localhost:8000/ui/
48	+	osmedeus server
49	+	# login with credentials from `~/.osmedeus/config.yaml`
50	+
51	+	# Delete workspace
52	+	osmedeus config delete -w workspace_name
53	+	```
54	+
55	+	## 💬 Community & Discussion
56	+
57	+	Join Our Discord server [here](https://discord.gg/gy4SWhpaPU)
58	+
59	+	## 💎 Donation
60	+
61	+	Please check out [this page](https://docs.osmedeus.org/donation/) for couple donation methods here
62	+
63	+
64	+	## License
65	+
66	+	`Osmedeus` is made with ♥ by [@j3ssiejjj](https://twitter.com/j3ssiejjj) and it is released under the MIT license.
67	+

■ ■ ■ ■ ■ ■

cmd/cloud.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/distribute"
6	+	"io"
7	+	"os"
8	+	"path"
9	+	"strings"
10	+
11	+	"github.com/j3ssie/osmedeus/core"
12	+	"github.com/j3ssie/osmedeus/libs"
13	+	"github.com/j3ssie/osmedeus/utils"
14	+	"github.com/spf13/cobra"
15	+	)
16	+
17	+	func init() {
18	+	var cloudCmd = &cobra.Command{
19	+	Use: "cloud",
20	+	Short: "Run scan in Distributed Cloud mode",
21	+	Long: core.Banner(),
22	+	RunE: runCloud,
23	+	}
24	+
25	+	// core options
26	+	cloudCmd.Flags().StringVarP(&options.Cloud.Module, "module", "m", "", "module name for running")
27	+	cloudCmd.Flags().StringVarP(&options.Cloud.Flow, "flow", "f", "general", "Flow name for running (default: general)")
28	+	cloudCmd.Flags().StringVarP(&options.Cloud.Workspace, "workspace", "w", "", "Name of workspace (default is same as target)")
29	+	cloudCmd.Flags().StringSliceVarP(&options.Cloud.Params, "params", "p", []string{}, "Custom params -p='foo=bar' (Multiple -p flags are accepted)")
30	+
31	+	// chunk inputs
32	+	cloudCmd.Flags().BoolVar(&options.Cloud.EnablePrivateIP, "privateIP", false, "Enable Private IP")
33	+	cloudCmd.Flags().BoolVar(&options.Cloud.TargetAsFile, "as-file", false, "Run target as file (use -T targets.txt file instead of -t targets.txt at cloud instance)")
34	+	cloudCmd.Flags().StringVar(&options.Cloud.LocalSyncFolder, "rfolder", fmt.Sprintf("/root/.%s/workspaces/", libs.BINARY), "Remote Folder to sync back to local")
35	+
36	+	// commands on cloud
37	+	cloudCmd.Flags().IntVar(&options.Cloud.Threads, "threads", 1, "Concurrency level on remote cloud")
38	+	cloudCmd.Flags().StringVar(&options.Cloud.Extra, "extra", "", "append raw command after the command builder")
39	+	cloudCmd.Flags().StringVar(&options.Cloud.RawCommand, "cmd", "", "specific raw command and override everything (eg: --cmd 'curl {{.Target}}')")
40	+	cloudCmd.Flags().StringVar(&options.Cloud.CloudWait, "wait", "30m", "time to wait before next queue check")
41	+	cloudCmd.Flags().StringVar(&options.Cloud.ClearTime, "clear", "10m", "time to wait before next clear check")
42	+	cloudCmd.Flags().StringVar(&options.Cloud.TempTarget, "tempTargets", "/tmp/osm-tmp-inputs/", "Temp Folder to store targets file")
43	+
44	+	// mics option
45	+	cloudCmd.Flags().BoolVarP(&options.Cloud.CopyWorkspaceToGit, "gws", "G", false, "Enable Copy Workspace to Git (run -f sync after done)")
46	+	cloudCmd.Flags().BoolVarP(&options.Cloud.DisableLocalSync, "no-lsync", "z", false, "Disable sync back data to local machine")
47	+	cloudCmd.Flags().BoolVar(&options.Cloud.BackgroundRun, "bg", false, "Send command to instance without checking if process is done or not")
48	+	cloudCmd.Flags().BoolVar(&options.Cloud.EnableTerraform, "tf", false, "Use terraform to create cloud instance")
49	+	cloudCmd.Flags().BoolVar(&options.Cloud.NoDelete, "no-del", false, "Don't delete instance after done")
50	+	cloudCmd.Flags().IntVar(&options.Cloud.Retry, "retry", 10, "Number of retry when command is error")
51	+	cloudCmd.SetHelpFunc(CloudHelp)
52	+	RootCmd.AddCommand(cloudCmd)
53	+
54	+	}
55	+
56	+	func runCloud(cmd *cobra.Command, _ []string) error {
57	+	DBInit()
58	+	// parse some argument
59	+	threads, _ := cmd.Flags().GetInt("thread")
60	+	if threads > 1 \|\| options.Cloud.Threads <= 1 {
61	+	options.Cloud.Threads = threads
62	+	}
63	+	utils.InforF("%v %v by %v", strings.Title(libs.BINARY), libs.VERSION, libs.AUTHOR)
64	+	utils.InforF("Store log file to: %v", options.LogFile)
65	+
66	+	// get pre run commands
67	+	getPreRun(&options)
68	+
69	+	// change targets list if chunk mode enable
70	+	if options.Cloud.EnableChunk {
71	+	utils.InforF("Running cloud scan in chunk mode")
72	+	for _, target := range options.Scan.Inputs {
73	+	chunkTargets := HandleChunksInputs(target)
74	+	if len(chunkTargets) == 0 {
75	+	continue
76	+	}
77	+
78	+	distribute.InitCloud(options, chunkTargets)
79	+	// remove chunk inputs
80	+	utils.DebugF("Remove chunk inputs file")
81	+	for _, ctarget := range chunkTargets {
82	+	os.RemoveAll(ctarget)
83	+	}
84	+	}
85	+	return nil
86	+	}
87	+
88	+	distribute.InitCloud(options, options.Scan.Inputs)
89	+	return nil
90	+	}
91	+
92	+	func HandleChunksInputs(target string) []string {
93	+	var chunkTargets []string
94	+	utils.MakeDir(options.Cloud.ChunkInputs)
95	+
96	+	if !utils.FileExists(target) {
97	+	utils.ErrorF("error to split input file: %v", target)
98	+	return chunkTargets
99	+	}
100	+
101	+	if options.Cloud.NumberOfParts == 0 {
102	+	options.Cloud.NumberOfParts = options.Concurrency
103	+	}
104	+
105	+	utils.DebugF("Splitting %v to %v part", target, options.Cloud.NumberOfParts)
106	+	rawChunks, err := utils.SplitLineChunks(target, options.Cloud.NumberOfParts)
107	+	if err != nil \|\| len(rawChunks) == 0 {
108	+	utils.ErrorF("error to split input file: %v", target)
109	+	return chunkTargets
110	+	}
111	+	fp, err := os.Open(target)
112	+	if err != nil {
113	+	utils.ErrorF("error to open input file: %v", target)
114	+	return chunkTargets
115	+	}
116	+	for index, offset := range rawChunks {
117	+	targetName := fmt.Sprintf("%s-chunk-%v", utils.CleanPath(target), index)
118	+	targetName = path.Join(options.Cloud.ChunkInputs, targetName)
119	+
120	+	sectionReader := io.NewSectionReader(fp, offset.Start, offset.Stop-offset.Start)
121	+	targetFile, err := os.OpenFile(targetName, os.O_RDWR\|os.O_CREATE\|os.O_TRUNC, 0644)
122	+	if err != nil {
123	+	utils.ErrorF("error when create chunk file: %v", target)
124	+	continue
125	+	}
126	+
127	+	_, err = io.Copy(targetFile, sectionReader)
128	+	if err != nil {
129	+	utils.ErrorF("error to read chunk file: %s", err)
130	+	continue
131	+	}
132	+	targetFile.Close()
133	+	chunkTargets = append(chunkTargets, targetName)
134	+	}
135	+
136	+	return chunkTargets
137	+	}
138	+
139	+	func getPreRun(options *libs.Options) {
140	+	if options.Cloud.Module != "" {
141	+	module := core.DirectSelectModule(*options, options.Cloud.Module)
142	+	if module == "" {
143	+	utils.ErrorF("Error to select module: %s", options.Cloud.Module)
144	+	return
145	+	}
146	+	parsedModule, err := core.ParseModules(module)
147	+	if err == nil {
148	+	options.Cloud.RemotePreRun = parsedModule.RemotePreRun
149	+	options.Cloud.LocalPostRun = parsedModule.LocalPostRun
150	+	options.Cloud.LocalPreRun = parsedModule.LocalPreRun
151	+	options.Cloud.LocalSteps = parsedModule.LocalSteps
152	+	}
153	+	return
154	+	}
155	+
156	+	if options.Cloud.Flow != "" {
157	+	flows := core.SelectFlow(options.Cloud.Flow, *options)
158	+	for _, flow := range flows {
159	+	parseFlow, err := core.ParseFlow(flow)
160	+	if err == nil {
161	+	options.Cloud.RemotePreRun = parseFlow.RemotePreRun
162	+	options.Cloud.LocalPostRun = parseFlow.LocalPostRun
163	+	options.Cloud.LocalPreRun = parseFlow.LocalPreRun
164	+	}
165	+	}
166	+	}
167	+	}
168	+

■ ■ ■ ■ ■ ■

cmd/config.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/database"
6	+	"github.com/j3ssie/osmedeus/execution"
7	+	"os"
8	+	"sort"
9	+
10	+	"github.com/j3ssie/osmedeus/core"
11	+	"github.com/j3ssie/osmedeus/utils"
12	+	"github.com/spf13/cobra"
13	+	)
14	+
15	+	func init() {
16	+	var configCmd = &cobra.Command{
17	+	Use: "config",
18	+	Short: "Do some config stuff",
19	+	Long: core.Banner(),
20	+	RunE: runConfig,
21	+	}
22	+
23	+	configCmd.Flags().StringP("action", "a", "", "Action")
24	+	configCmd.Flags().StringP("pluginsRepo", "p", "[email protected]:j3ssie/osmedeus-plugins.git", "Osmedeus Plugins repository")
25	+	// for cred action
26	+	configCmd.Flags().String("user", "", "Username")
27	+	configCmd.Flags().String("pass", "", "Password")
28	+	configCmd.Flags().StringP("workspace", "w", "", "Name of workspace")
29	+
30	+	configCmd.SetHelpFunc(ConfigHelp)
31	+	RootCmd.AddCommand(configCmd)
32	+	}
33	+
34	+	func runConfig(cmd *cobra.Command, args []string) error {
35	+	sort.Strings(args)
36	+	action, _ := cmd.Flags().GetString("action")
37	+	//pluginsRepo, _ := cmd.Flags().GetString("pluginsRepo")
38	+	workspace, _ := cmd.Flags().GetString("workspace")
39	+	DBInit()
40	+
41	+	// backward compatible
42	+	if action == "" && len(args) > 0 {
43	+	action = args[0]
44	+	}
45	+
46	+	switch action {
47	+	case "init":
48	+	if utils.FolderExists(fmt.Sprintf("%vcore", options.Env.RootFolder)) {
49	+	utils.GoodF("Look like you got properly setup.")
50	+	}
51	+	break
52	+	case "cred":
53	+	username, _ := cmd.Flags().GetString("user")
54	+	password, _ := cmd.Flags().GetString("pass")
55	+	//database.CreateUser(username, password)
56	+	utils.GoodF("Create new credentials %v:%v \n", username, password)
57	+	break
58	+
59	+	case "reload":
60	+	core.ReloadConfig(options)
61	+	break
62	+
63	+	case "delete":
64	+	options.Scan.Input = workspace
65	+	options.Scan.ROptions = core.ParseInput(options.Scan.Input, options)
66	+	utils.InforF("Delete Workspace: %v", options.Scan.ROptions["Workspace"])
67	+	os.RemoveAll(options.Scan.ROptions["Output"])
68	+	//ws := database.SelectScan(options.Scan.ROptions["Workspace"])
69	+	//database.DeleteScan(int(ws.ID))
70	+	break
71	+
72	+	case "pull":
73	+	for repo := range options.Storages {
74	+	execution.PullResult(repo, options)
75	+	}
76	+	break
77	+
78	+	case "update":
79	+	core.Update(options)
80	+	break
81	+
82	+	case "clean", "cl", "c":
83	+	database.ClearDB()
84	+	break
85	+	}
86	+
87	+	return nil
88	+	}
89	+

■ ■ ■ ■ ■ ■

cmd/exec.go

1	+	package cmd
2	+
3	+	import (
4	+	"github.com/j3ssie/osmedeus/core"
5	+	"github.com/j3ssie/osmedeus/utils"
6	+	"github.com/spf13/cobra"
7	+	"os"
8	+	)
9	+
10	+	func init() {
11	+	var execCmd = &cobra.Command{
12	+	Use: "exec",
13	+	Short: "Execute inline osmedeus scripts",
14	+	Long: core.Banner(),
15	+	RunE: runExec,
16	+	}
17	+
18	+	execCmd.Flags().String("script", "", "Scripts to run (Multiple -s flags are accepted)")
19	+	execCmd.Flags().StringP("scriptFile", "S", "", "File contain list of scripts")
20	+	RootCmd.AddCommand(execCmd)
21	+	}
22	+
23	+	func runExec(cmd *cobra.Command, _ []string) error {
24	+	script, _ := cmd.Flags().GetString("script")
25	+	scriptFile, _ := cmd.Flags().GetString("scriptFile")
26	+
27	+	var scripts []string
28	+	if script != "" {
29	+	scripts = append(scripts, script)
30	+	}
31	+	if scriptFile != "" {
32	+	moreScripts := utils.ReadingFileUnique(scriptFile)
33	+	if len(moreScripts) > 0 {
34	+	scripts = append(scripts, moreScripts...)
35	+	}
36	+	}
37	+
38	+	if len(scripts) == 0 {
39	+	utils.ErrorF("No scripts provided")
40	+	os.Exit(0)
41	+	}
42	+	runner, _ := core.InitRunner("example.com", options)
43	+
44	+	for _, t := range options.Scan.Inputs {
45	+	// start to run scripts
46	+	options.Scan.ROptions = core.ParseInput(t, options)
47	+	for _, script := range scripts {
48	+	script = core.ResolveData(script, options.Scan.ROptions)
49	+	utils.InforF("Script: %v", script)
50	+	runner.RunScript(script, options)
51	+	}
52	+	}
53	+
54	+	return nil
55	+	}
56	+

■ ■ ■ ■ ■ ■

cmd/health.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/fatih/color"
6	+	"github.com/j3ssie/osmedeus/core"
7	+	"github.com/j3ssie/osmedeus/execution"
8	+	"github.com/j3ssie/osmedeus/libs"
9	+	"github.com/j3ssie/osmedeus/utils"
10	+	"github.com/spf13/cobra"
11	+	"path"
12	+	)
13	+
14	+	func init() {
15	+	var execCmd = &cobra.Command{
16	+	Use: "health",
17	+	Aliases: []string{"hea", "heal", "health", "healht"},
18	+	Short: "Run diagnostics to check configurations",
19	+	Long: core.Banner(),
20	+	RunE: runHealth,
21	+	}
22	+	RootCmd.AddCommand(execCmd)
23	+	}
24	+
25	+	func runHealth(_ *cobra.Command, _ []string) error {
26	+	if options.PremiumPackage {
27	+	fmt.Printf("💠 Osmedeus %s: Run diagnostics to check if everything okay\n", libs.VERSION)
28	+	} else {
29	+	fmt.Printf("🚀 Osmedeus %s: Run diagnostics to check if everything okay\n", libs.VERSION)
30	+	}
31	+
32	+	err := checkCorePrograms(options)
33	+	if err != nil {
34	+	fmt.Printf("‼️ There is might be something wrong with your setup: %v\n", err)
35	+	return nil
36	+	}
37	+
38	+	err = listFlows(options)
39	+	if err != nil {
40	+	fmt.Printf("‼️ There is might be something wrong with your setup: %v\n", err)
41	+	return nil
42	+	}
43	+
44	+	fmt.Printf(color.GreenString("\n🦾 It’s all good. Happy Hacking 🦾\n"))
45	+	return nil
46	+	}
47	+
48	+	func checkCorePrograms(options libs.Options) error {
49	+	exist := utils.FolderExists(options.Env.BaseFolder)
50	+	if !exist {
51	+	color.Red("[-] Core folder setup incorrect: %v", options.Env.BaseFolder)
52	+	return fmt.Errorf("error running diagnostics")
53	+	}
54	+
55	+	// check core programs
56	+	var err error
57	+	_, err = utils.RunCommandWithErr("jaeles -h")
58	+	if err != nil {
59	+	color.Red("[-] Core program setup incorrectly")
60	+	return fmt.Errorf("error checking core programs: %v", "jaeles")
61	+
62	+	}
63	+	_, err = utils.RunCommandWithErr("amass -h")
64	+	if err != nil {
65	+	color.Red("[-] Core program setup incorrectly")
66	+	return fmt.Errorf("error checking core programs: %v", "amass")
67	+
68	+	}
69	+	_, err = utils.RunCommandWithErr(fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "httprobe")))
70	+	if err != nil {
71	+	color.Red("[-] Core program setup incorrectly")
72	+	return fmt.Errorf("error checking core programs: %v", fmt.Sprintf("%s -h", path.Join(options.Env.BinariesFolder, "httprobe")))
73	+
74	+	}
75	+	fmt.Printf("[+] Health Check Core Programs: %s\n", color.GreenString("✔"))
76	+
77	+	// Check core signatures
78	+	okVuln := false
79	+	if utils.DirLength("~/.jaeles/base-signatures/") > 0 \|\| utils.DirLength("~/pro-signatures/") > 0 {
80	+	okVuln = true
81	+	}
82	+
83	+	if utils.DirLength("~/nuclei-templates") > 0 {
84	+	okVuln = true
85	+	}
86	+
87	+	if okVuln {
88	+	fmt.Printf("[+] Health Check Vulnerability scanning config: %s\n", color.GreenString("✔"))
89	+	} else {
90	+	color.Red("vulnerability scanning config setup incorrectly")
91	+	return fmt.Errorf("vulnerability scanning config setup incorrectly")
92	+	}
93	+
94	+	// check data folder
95	+	if utils.FolderExists(options.Env.DataFolder) {
96	+	fmt.Printf("[+] Health Check Data Config: %s\n", color.GreenString("✔"))
97	+	} else {
98	+	color.Red("[-] Data setup incorrectly: %v", options.Env.DataFolder)
99	+	return fmt.Errorf("[-] Data setup incorrectly: %v", options.Env.DataFolder)
100	+	}
101	+
102	+	// check cloud config
103	+	var okCloud bool
104	+	if utils.FileExists(path.Join(options.Env.CloudConfigFolder, "config.yaml")) {
105	+	okCloud = true
106	+	if utils.DirLength(path.Join(options.Env.CloudConfigFolder, "providers")) == 0 {
107	+	okCloud = false
108	+	}
109	+	if utils.DirLength(path.Join(options.Env.CloudConfigFolder, "ssh")) < 2 {
110	+	okCloud = false
111	+	}
112	+	if okCloud {
113	+	fmt.Printf("[+] Health Check Cloud Config: %s\n", color.GreenString("✔"))
114	+	} else {
115	+	fmt.Printf(color.YellowString("%s If you install osmedeus on a single machine then it's okay to ignore the cloud setup\n", "[!] Cloud config setup incorrectly."))
116	+	}
117	+
118	+	}
119	+
120	+	if execution.ValidGitURL(options.Storages["summary_repo"]) {
121	+	if utils.DirLength(options.Storages["summary_storage"]) > 1 {
122	+	fmt.Printf("[+] Health Check Storages Config: %s\n", color.GreenString("✔"))
123	+	}
124	+	}
125	+
126	+	return nil
127	+	}
128	+
129	+	func listFlows(options libs.Options) error {
130	+	flows := core.ListFlow(options)
131	+	if len(flows) == 0 {
132	+	color.Red("[-] Error to list workflows: %s", options.Env.WorkFlowsFolder)
133	+	return fmt.Errorf("[-] Error to list workflows: %s", options.Env.WorkFlowsFolder)
134	+	}
135	+	fmt.Printf("[+] Health Check Workflows: %s\n", color.GreenString("✔"))
136	+
137	+	fmt.Printf("\nChecking available workflow at: %s \n\n", color.HiBlueString(options.Env.WorkFlowsFolder))
138	+	for _, flow := range flows {
139	+	parsedFlow, err := core.ParseFlow(flow)
140	+	if err != nil {
141	+	utils.ErrorF("Error parsing flow: %v", flow)
142	+	}
143	+	fmt.Printf("%10s - %s\n", parsedFlow.Name, parsedFlow.Desc)
144	+	}
145	+	h := "\nUsage:\n"
146	+	h += " osmedeus scan -f [flowName] -t [target] \n"
147	+	fmt.Printf(h)
148	+	return nil
149	+	}
150	+

■ ■ ■ ■ ■ ■

cmd/provider.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/core"
6	+	"github.com/j3ssie/osmedeus/distribute"
7	+	"github.com/j3ssie/osmedeus/provider"
8	+	"github.com/j3ssie/osmedeus/utils"
9	+	"github.com/spf13/cobra"
10	+	"sort"
11	+	)
12	+
13	+	func init() {
14	+	var providerCmd = &cobra.Command{
15	+	Use: "provider",
16	+	Short: "Cloud utils for Distributed Mode",
17	+	Long: core.Banner(),
18	+	RunE: runProvider,
19	+	}
20	+
21	+	providerCmd.PersistentFlags().StringVar(&options.Cloud.RawCommand, "cmd", "", "raw command")
22	+	providerCmd.PersistentFlags().StringVar(&options.Cloud.CloudWait, "wait", "30m", "timeout to wait before next queue check")
23	+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.CheckingLimit, "check", false, "Only check for limit of config")
24	+	providerCmd.PersistentFlags().StringVar(&options.Cloud.InstanceName, "name", "", "override instance name")
25	+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.BackgroundRun, "bg", false, "Send command to instance and run it in background")
26	+	providerCmd.PersistentFlags().BoolVar(&options.Cloud.IgnoreConfigFile, "ic", false, "Ignore token in the config file")
27	+	providerCmd.PersistentFlags().IntVar(&options.Cloud.Retry, "retry", 8, "Number of retry when command is error")
28	+
29	+	var providerBuild = &cobra.Command{
30	+	Use: "build",
31	+	Short: "Build cloud image",
32	+	Long: core.Banner(),
33	+	RunE: runProviderBuild,
34	+	}
35	+	var providerCreate = &cobra.Command{
36	+	Use: "create",
37	+	Short: "Create cloud instance based on image",
38	+	Long: core.Banner(),
39	+	RunE: runProviderCreate,
40	+	}
41	+
42	+	var healthCmd = &cobra.Command{
43	+	Use: "health",
44	+	Short: "Cloud Utility to check cloud instance health",
45	+	Long: core.Banner(),
46	+	RunE: runCloudHealth,
47	+	}
48	+
49	+	var providerValidate = &cobra.Command{
50	+	Use: "validate",
51	+	Short: "Run various action on cloud provider",
52	+	Long: core.Banner(),
53	+	RunE: runProviderValidate,
54	+	}
55	+	providerCmd.AddCommand(providerValidate)
56	+
57	+	providerCmd.AddCommand(healthCmd)
58	+	providerCmd.AddCommand(providerCreate)
59	+	providerCmd.AddCommand(providerBuild)
60	+	providerCmd.AddCommand(providerValidate)
61	+	providerCmd.SetHelpFunc(CloudHelp)
62	+	RootCmd.AddCommand(providerCmd)
63	+	}
64	+
65	+	func runCloudHealth(_ *cobra.Command, _ []string) error {
66	+	DBInit()
67	+	distribute.CheckingCloudInstance(options)
68	+	return nil
69	+	}
70	+
71	+	func runProvider(_ *cobra.Command, args []string) error {
72	+	DBInit()
73	+	if len(args) == 0 {
74	+	fmt.Println(CloudUsage())
75	+	}
76	+	return nil
77	+	}
78	+
79	+	func runProviderBuild(_ *cobra.Command, _ []string) error {
80	+	options.Cloud.OnlyCreateDroplet = true
81	+	options.Cloud.ReBuildBaseImage = true
82	+	distribute.InitCloud(options, options.Scan.Inputs)
83	+	return nil
84	+	}
85	+
86	+	func runProviderCreate(_ *cobra.Command, _ []string) error {
87	+	options.Cloud.OnlyCreateDroplet = true
88	+	if len(options.Scan.Inputs) == 0 {
89	+	options.Scan.Inputs = append(options.Scan.Inputs, utils.RandomString(4))
90	+	}
91	+
92	+	distribute.InitCloud(options, options.Scan.Inputs)
93	+	return nil
94	+	}
95	+
96	+	func runProviderValidate(_ *cobra.Command, actions []string) error {
97	+	//options.Cloud.IgnoreSetup = true
98	+	clouds := distribute.GetClouds(options)
99	+	sort.Strings(actions)
100	+
101	+	for _, cloud := range clouds {
102	+	for _, action := range actions {
103	+	err := cloud.Provider.Action(provider.ListInstance)
104	+	if err != nil {
105	+	utils.ErrorF("error running %v ", action)
106	+	}
107	+	for _, instance := range cloud.Provider.Instances {
108	+	fmt.Printf("[%v]: %v -- %v", instance.ProviderName, instance.InstanceID, instance.IPAddress)
109	+	}
110	+	}
111	+	}
112	+	return nil
113	+	}
114	+

■ ■ ■ ■ ■ ■

cmd/root.go

1	+	package cmd
2	+
3	+	import (
4	+	"bufio"
5	+	"fmt"
6	+	"github.com/j3ssie/osmedeus/database"
7	+	"os"
8	+	"strings"
9	+
10	+	"github.com/j3ssie/osmedeus/core"
11	+	"github.com/j3ssie/osmedeus/libs"
12	+	"github.com/j3ssie/osmedeus/utils"
13	+	"github.com/spf13/cobra"
14	+	"gorm.io/gorm"
15	+	)
16	+
17	+	var options = libs.Options{}
18	+
19	+	// DB database variables
20	+	var DB *gorm.DB
21	+
22	+	var RootCmd = &cobra.Command{
23	+	Use: fmt.Sprintf("%s", libs.BINARY),
24	+	Short: fmt.Sprintf("%s - %s", libs.BINARY, libs.DESC),
25	+	Long: core.Banner(),
26	+	}
27	+
28	+	// Execute main function
29	+	func Execute() {
30	+	if err := RootCmd.Execute(); err != nil {
31	+	fmt.Println(err)
32	+	os.Exit(1)
33	+	}
34	+	}
35	+
36	+	func init() {
37	+	RootCmd.PersistentFlags().StringVar(&options.Env.RootFolder, "rootFolder", fmt.Sprintf("~/.%s/", libs.BINARY), "Root Folder to store Result")
38	+	RootCmd.PersistentFlags().StringVar(&options.Env.BaseFolder, "baseFolder", fmt.Sprintf("~/%s-base/", libs.BINARY), "Base Folder which is store data, binaries and workflows")
39	+	RootCmd.PersistentFlags().StringVar(&options.ConfigFile, "configFile", fmt.Sprintf("~/.%s/config.yaml", libs.BINARY), "Config File")
40	+	RootCmd.PersistentFlags().StringVar(&options.LogFile, "log", "", "Log File")
41	+	RootCmd.PersistentFlags().IntVarP(&options.Concurrency, "concurrency", "c", 1, "Concurrency level (recommend to keep it as 1 on machine has RAM smaller than 2GB)")
42	+
43	+	// parse target as global flag
44	+	RootCmd.PersistentFlags().StringSliceVarP(&options.Scan.Inputs, "target", "t", []string{}, "Target to running")
45	+	RootCmd.PersistentFlags().StringVarP(&options.Scan.InputList, "targets", "T", "", "List of target as a file")
46	+
47	+	// cloud flags
48	+	RootCmd.PersistentFlags().BoolVar(&options.Cloud.EnableChunk, "chunk", false, "Enable chunk mode")
49	+	RootCmd.PersistentFlags().IntVarP(&options.Cloud.NumberOfParts, "chunk-parts", "P", 0, "Number of chunks file to split (default: equal with concurrency)")
50	+	RootCmd.PersistentFlags().StringVar(&options.Cloud.ChunkInputs, "chunkFolder", "/tmp/chunk-inputs/", "Temp Folder to store chunk inputs")
51	+	RootCmd.PersistentFlags().StringVar(&options.Timeout, "timeout", "", "Global timeout for each step (e.g: 60s, 30m, 2h)")
52	+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Size, "size", "", "Override Size of cloud instance (default will get from 'cloud/config.yaml')")
53	+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Region, "region", "", "Override Region of cloud instance (default will get from 'cloud/config.yaml')")
54	+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Token, "token", "", "Override token of cloud instance (default will get from 'cloud/config.yaml')")
55	+	RootCmd.PersistentFlags().StringVar(&options.Cloud.Provider, "provider", "", "Override provider of cloud instance (default will get from 'cloud/config.yaml')")
56	+	RootCmd.PersistentFlags().BoolVar(&options.Cloud.ReBuildBaseImage, "rebuild", false, "Forced to rebuild the images event though the version didn't change")
57	+
58	+	// mics option
59	+	RootCmd.PersistentFlags().StringVarP(&options.ScanID, "sid", "s", "", "Scan ID to continue the scan without create new scan record")
60	+	RootCmd.PersistentFlags().BoolVarP(&options.Resume, "resume", "R", false, "Enable Resume")
61	+	RootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Enable Debug output")
62	+	RootCmd.PersistentFlags().BoolVarP(&options.Quite, "quite", "q", false, "Show only essential information")
63	+	RootCmd.PersistentFlags().BoolVar(&options.WildCardCheck, "ww", false, "Check for wildcard target")
64	+	RootCmd.PersistentFlags().BoolVar(&options.DisableValidateInput, "nv", false, "Disable Validate Input")
65	+	RootCmd.PersistentFlags().BoolVar(&options.Update.NoUpdate, "nu", false, "Disable Update options")
66	+	RootCmd.PersistentFlags().BoolVarP(&options.Verbose, "verbose", "V", false, "Show stat info too")
67	+	RootCmd.PersistentFlags().BoolVarP(&options.EnableFormatInput, "format-input", "J", false, "Enable special input format")
68	+
69	+	// disable options
70	+	RootCmd.PersistentFlags().BoolVar(&options.NoNoti, "nn", false, "No notification")
71	+	RootCmd.PersistentFlags().BoolVar(&options.NoBanner, "nb", false, "No banner")
72	+	RootCmd.PersistentFlags().BoolVarP(&options.NoDB, "no-db", "D", false, "No store DB record")
73	+	RootCmd.PersistentFlags().BoolVarP(&options.NoGit, "no-git", "N", false, "No git storage")
74	+	RootCmd.PersistentFlags().BoolVarP(&options.NoClean, "no-clean", "C", false, "No clean junk output")
75	+	RootCmd.PersistentFlags().StringSliceVarP(&options.Exclude, "exclude", "x", []string{}, "Exclude module name (Multiple -x flags are accepted)")
76	+	RootCmd.PersistentFlags().BoolVarP(&options.CustomGit, "git", "g", false, "Use custom Git repo")
77	+
78	+	// sync options
79	+	RootCmd.PersistentFlags().BoolVar(&options.EnableDeStorage, "des", false, "Enable Dedicated Storages")
80	+	RootCmd.PersistentFlags().BoolVar(&options.GitSync, "sync", false, "Enable Sync Check before doing git push")
81	+	RootCmd.PersistentFlags().IntVar(&options.SyncTimes, "sync-timee", 15, "Number of times to check before force push")
82	+	RootCmd.PersistentFlags().IntVar(&options.PollingTime, "poll-timee", 100, "Number of seconds to sleep before do next sync check")
83	+	RootCmd.PersistentFlags().BoolVar(&options.NoCdn, "no-cdn", false, "Disable CDN feature")
84	+	RootCmd.PersistentFlags().BoolVarP(&options.EnableBackup, "backup", "b", false, "Enable Backup after done")
85	+	RootCmd.PersistentFlags().BoolVar(&options.JsonOutput, "json", false, "Output as JSON")
86	+
87	+	// update options
88	+	RootCmd.PersistentFlags().BoolVar(&options.Update.IsUpdateBin, "bin", false, "Update binaries too")
89	+	RootCmd.PersistentFlags().BoolVar(&options.Update.EnableUpdate, "update", false, "Enable auto update")
90	+	RootCmd.PersistentFlags().StringVar(&options.Update.UpdateFolder, "update-folder", "/tmp/osm-update", "Folder to clone the update folder")
91	+
92	+	RootCmd.SetHelpFunc(RootHelp)
93	+	cobra.OnInitialize(initConfig)
94	+	}
95	+
96	+	// initConfig reads in config file and ENV variables if set.
97	+	func initConfig() {
98	+	if options.JsonOutput {
99	+	options.Quite = true
100	+	}
101	+
102	+	/* Really Start the program */
103	+	utils.InitLog(&options)
104	+	core.InitConfig(&options)
105	+
106	+	// parse inputs
107	+	if options.Scan.InputList != "" {
108	+	if utils.FileExists(options.Scan.InputList) {
109	+	options.Scan.Inputs = append(options.Scan.Inputs, utils.ReadingFileUnique(options.Scan.InputList)...)
110	+	}
111	+	}
112	+
113	+	// detect if anything came from stdin
114	+	stat, _ := os.Stdin.Stat()
115	+	if (stat.Mode() & os.ModeCharDevice) == 0 {
116	+	sc := bufio.NewScanner(os.Stdin)
117	+	for sc.Scan() {
118	+	target := strings.TrimSpace(sc.Text())
119	+	if err := sc.Err(); err == nil && target != "" {
120	+	options.Scan.Inputs = append(options.Scan.Inputs, target)
121	+	}
122	+	}
123	+	}
124	+
125	+	if options.Update.EnableUpdate && options.Update.NoUpdate == false {
126	+	core.Update(options)
127	+	}
128	+	}
129	+
130	+	// DBInit init database connection
131	+	func DBInit() {
132	+	var err error
133	+	DB, err = database.InitDB(options)
134	+	if err != nil {
135	+	// simple retry
136	+	DB, err = database.InitDB(options)
137	+	if err != nil {
138	+	fmt.Printf("[panic] Can't connect to DB at %v\n", options.Server.DBPath)
139	+	os.Exit(-1)
140	+	}
141	+	}
142	+	}
143	+

■ ■ ■ ■ ■ ■

cmd/scan.go

1	+	package cmd
2	+
3	+	import (
4	+	"github.com/j3ssie/osmedeus/core"
5	+	"github.com/j3ssie/osmedeus/libs"
6	+	"github.com/j3ssie/osmedeus/utils"
7	+	"github.com/panjf2000/ants"
8	+	"github.com/spf13/cobra"
9	+	"strings"
10	+	"sync"
11	+	)
12	+
13	+	func init() {
14	+	var scanCmd = &cobra.Command{
15	+	Use: "scan",
16	+	Short: "Do Scan based on predefined flow",
17	+	Long: core.Banner(),
18	+	RunE: runScan,
19	+	}
20	+
21	+	scanCmd.Flags().StringSliceVarP(&options.Scan.Modules, "module", "m", []string{}, "Target to running")
22	+	scanCmd.Flags().StringVarP(&options.Scan.Flow, "flow", "f", "general", "Flow name for running (default: general)")
23	+	scanCmd.Flags().StringVarP(&options.Scan.CustomWorkspace, "workspace", "w", "", "Name of workspace (default is same as target)")
24	+	scanCmd.Flags().StringSliceVarP(&options.Scan.Params, "params", "p", []string{}, "Custom params -p='foo=bar' (Multiple -p flags are accepted)")
25	+	scanCmd.SetHelpFunc(ScanHelp)
26	+	RootCmd.AddCommand(scanCmd)
27	+	}
28	+
29	+	func runScan(_ *cobra.Command, _ []string) error {
30	+	DBInit()
31	+	utils.InforF("%v %v by %v", strings.Title(libs.BINARY), libs.VERSION, libs.AUTHOR)
32	+	utils.InforF("Store log file to: %v", options.LogFile)
33	+
34	+	var wg sync.WaitGroup
35	+	p, _ := ants.NewPoolWithFunc(options.Concurrency, func(i interface{}) {
36	+	// really start to scan
37	+	CreateRunner(i)
38	+	wg.Done()
39	+	}, ants.WithPreAlloc(true))
40	+	defer p.Release()
41	+
42	+	if options.Cloud.EnableChunk {
43	+	for _, target := range options.Scan.Inputs {
44	+	chunkTargets := HandleChunksInputs(target)
45	+	for _, chunkTarget := range chunkTargets {
46	+	wg.Add(1)
47	+	_ = p.Invoke(chunkTarget)
48	+	}
49	+	}
50	+	} else {
51	+	for _, target := range options.Scan.Inputs {
52	+	wg.Add(1)
53	+	_ = p.Invoke(strings.TrimSpace(target))
54	+	}
55	+	}
56	+
57	+	wg.Wait()
58	+	return nil
59	+	}
60	+
61	+	func CreateRunner(j interface{}) {
62	+	target := j.(string)
63	+	runner, err := core.InitRunner(target, options)
64	+	if err != nil {
65	+	utils.ErrorF("Error init runner with: %s", target)
66	+	return
67	+	}
68	+	runner.Start()
69	+	}
70	+

■ ■ ■ ■ ■ ■

cmd/server.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+
6	+	"github.com/j3ssie/osmedeus/core"
7	+	"github.com/j3ssie/osmedeus/server"
8	+	"github.com/spf13/cobra"
9	+	)
10	+
11	+	func init() {
12	+	var serverCmd = &cobra.Command{
13	+	Use: "server",
14	+	Short: "Start Web Server",
15	+	Long: core.Banner(),
16	+	RunE: runServer,
17	+	}
18	+	serverCmd.Flags().String("host", "0.0.0.0", "IP address to bind the server")
19	+	serverCmd.Flags().String("port", "8000", "Port")
20	+	serverCmd.Flags().IntVar(&options.Server.PollingTime, "poll-time", 60, "Polling time to check next task")
21	+	serverCmd.Flags().BoolVar(&options.Server.DisableSSL, "disable-ssl", false, "Disable workspaces directory listing")
22	+	serverCmd.Flags().BoolVar(&options.Server.DisableWorkspaceListing, "disable-listing", false, "Disable workspaces directtory listing")
23	+	serverCmd.Flags().BoolVar(&options.Server.PreFork, "prefork", false, "Enable Prefork mode for api server")
24	+	RootCmd.AddCommand(serverCmd)
25	+	}
26	+
27	+	func runServer(cmd *cobra.Command, _ []string) error {
28	+	host, _ := cmd.Flags().GetString("host")
29	+	port, _ := cmd.Flags().GetString("port")
30	+	options.Server.Bind = fmt.Sprintf("%v:%v", host, port)
31	+	DBInit()
32	+
33	+	// create default user
34	+	//database.CreateUser(options.Client.Username, options.Client.Password)
35	+
36	+	server.StartServer(options)
37	+	return nil
38	+	}
39	+

■ ■ ■ ■ ■ ■

cmd/update.go

1	+	package cmd
2	+
3	+	import (
4	+	"github.com/j3ssie/osmedeus/core"
5	+	"github.com/spf13/cobra"
6	+	)
7	+
8	+	func init() {
9	+	var updateCmd = &cobra.Command{
10	+	Use: "update",
11	+	Short: "Check latest Update",
12	+	Long: core.Banner(),
13	+	RunE: runUpdate,
14	+	}
15	+	updateCmd.Flags().String("repo", "", "Update repository URL")
16	+	RootCmd.AddCommand(updateCmd)
17	+	}
18	+
19	+	func runUpdate(cmd *cobra.Command, _ []string) error {
20	+	repo, _ := cmd.Flags().GetString("repo")
21	+	if repo != "" {
22	+	options.Update.UpdateURL = repo
23	+	}
24	+
25	+	core.Update(options)
26	+	return nil
27	+	}
28	+

■ ■ ■ ■ ■ ■

cmd/usage.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/fatih/color"
6	+	"github.com/j3ssie/osmedeus/core"
7	+	"github.com/j3ssie/osmedeus/libs"
8	+	"github.com/spf13/cobra"
9	+	)
10	+
11	+	// RootUsage base help
12	+	func RootUsage() {
13	+	var h string
14	+	h += ScanUsage()
15	+	h += CloudUsage()
16	+	h += UtilsUsage()
17	+
18	+	fmt.Println(h)
19	+	}
20	+
21	+	func ScanUsage() string {
22	+	h := color.HiCyanString("\nScan Usage:\n")
23	+	h += " osmedeus scan -f [flowName] -t [target] \n"
24	+	h += " osmedeus scan -m [modulePath] -T [targetsFile] \n"
25	+	h += " osmedeus scan -f /path/to/flow.yaml -t [target] \n"
26	+	h += " osmedeus scan -m /path/to/module.yaml -t [target] --params 'port=9200'\n"
27	+	h += " osmedeus scan -m /path/to/module.yaml -t [target] -l /tmp/log.log\n"
28	+	h += " cat targets \| osmedeus scan -f sample\n"
29	+
30	+	h += color.HiCyanString("\nPractical Scan Usage:\n")
31	+	h += " osmedeus scan -T list_of_targets.txt -W custom_workspaces\n"
32	+	h += " osmedeus scan -t target.com -w workspace_name --debug\n"
33	+	h += " osmedeus scan -f general -t www.sample.com\n"
34	+	h += " osmedeus scan -f gdirb -T list_of_target.txt\n"
35	+	h += " osmedeus scan -m ~/.osmedeus/core/workflow/test/dirbscan.yaml -t list_of_urls.txt\n"
36	+
37	+	return h
38	+	}
39	+
40	+	func UtilsUsage() string {
41	+	h := color.HiCyanString("\nUtilities Usage:\n")
42	+	h += " osmedeus health \n"
43	+	h += " osmedeus version --json \n"
44	+	h += " osmedeus utils tmux ls \n"
45	+	h += " osmedeus utils tmux logs -A -l 10 \n"
46	+	h += " osmedeus utils ps \n"
47	+	h += " osmedeus utils ps --proc 'jaeles' \n"
48	+	h += " osmedeus utils cron --cmd 'osmdeus scan -t example.com' --sch 60\n"
49	+	h += " osmedeus utils cron --for --cmd 'osmedeus scan -t example.com'\n"
50	+	return h
51	+	}
52	+
53	+	func ConfigUsage() string {
54	+	h := color.HiCyanString("\nConfig Usage:\n")
55	+	h += " osmedeus config [action] [OPTIONS] \n"
56	+	h += " osmedeus config init -p https://github.com/j3ssie/osmedeus-plugins\n"
57	+	h += " osmedeus config --user newusser --pass newpassword\n"
58	+	h += " osmedeus config reload \n"
59	+	h += " osmedeus config update \n"
60	+	h += " osmedeus config clean \n"
61	+	h += " osmedeus config delete -t woskapce \n"
62	+	h += " osmedeus config delete -w workspace_name \n"
63	+	return h
64	+	}
65	+
66	+	func CloudUsage() string {
67	+	h := color.HiCyanString("\nProvider Usage:\n")
68	+	h += " osmedeus provider build \n"
69	+	h += " osmedeus provider build --token xxx --rebuild --ic\n"
70	+	h += " osmedeus provider create --name 'sample' \n"
71	+	h += " osmedeus provider health --debug \n"
72	+
73	+	h += color.HiCyanString("\nCloud Usage:\n")
74	+	h += " osmedeus cloud -f [flowName] -t [target] \n"
75	+	h += " osmedeus cloud -m [modulePath] -t [target] \n"
76	+	h += " osmedeus cloud -c 10 -f [flowName] -T [targetsFile] \n"
77	+	h += " osmedeus cloud --token xxx -G -c 10 -f [flowName] -T [targetsFile] \n"
78	+	h += " osmedeus cloud --chunk -c 10 -f [flowName] -t [targetsFile] \n"
79	+
80	+	return h
81	+	}
82	+
83	+	// ScanHelp scan help message
84	+	func ScanHelp(cmd *cobra.Command, _ []string) {
85	+	fmt.Println(core.Banner())
86	+	fmt.Println(cmd.UsageString())
87	+	h := ScanUsage()
88	+	fmt.Println(h)
89	+	fmt.Printf("Documentation can be found here: %s\n", color.GreenString(libs.DOCS))
90	+	}
91	+
92	+	// CloudHelp scan help message
93	+	func CloudHelp(cmd *cobra.Command, _ []string) {
94	+	fmt.Println(core.Banner())
95	+	fmt.Println(cmd.UsageString())
96	+	h := CloudUsage()
97	+	fmt.Println(h)
98	+	fmt.Printf("Documentation can be found here: %s\n", color.GreenString(libs.DOCS))
99	+	}
100	+
101	+	// ConfigHelp config help message
102	+	func ConfigHelp(cmd *cobra.Command, _ []string) {
103	+	fmt.Println(core.Banner())
104	+	fmt.Println(cmd.UsageString())
105	+	h := ConfigUsage()
106	+	fmt.Println(h)
107	+	fmt.Printf("Documentation can be found here: %s\n", color.GreenString(libs.DOCS))
108	+	}
109	+
110	+	// UtilsHelp utils help message
111	+	func UtilsHelp(cmd *cobra.Command, _ []string) {
112	+	fmt.Println(core.Banner())
113	+	fmt.Println(cmd.UsageString())
114	+	h := UtilsUsage()
115	+	fmt.Println(h)
116	+	fmt.Printf("Documentation can be found here: %s\n", color.GreenString(libs.DOCS))
117	+	}
118	+
119	+	// RootHelp print help message
120	+	func RootHelp(cmd *cobra.Command, _ []string) {
121	+	fmt.Println(core.Banner())
122	+	fmt.Println(cmd.UsageString())
123	+	RootUsage()
124	+	fmt.Printf("Documentation can be found here: %s\n", color.GreenString(libs.DOCS))
125	+	}
126	+

■ ■ ■ ■ ■ ■

cmd/utils.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/fatih/color"
6	+	"github.com/j3ssie/osmedeus/core"
7	+	"github.com/j3ssie/osmedeus/execution"
8	+	"github.com/j3ssie/osmedeus/libs"
9	+	jsoniter "github.com/json-iterator/go"
10	+	"github.com/spf13/cobra"
11	+	)
12	+
13	+	func init() {
14	+	var utilsCmd = &cobra.Command{
15	+	Use: "utils",
16	+	Short: "Utils to get some information from the system",
17	+	Long: core.Banner(),
18	+	RunE: runProvider,
19	+	}
20	+
21	+	var psCmd = &cobra.Command{
22	+	Use: "ps",
23	+	Short: "Utility to get information about running process",
24	+	Long: core.Banner(),
25	+	RunE: runPs,
26	+	}
27	+	psCmd.Flags().StringSlice("proc", []string{}, "Process name")
28	+
29	+	var tmuxCmd = &cobra.Command{
30	+	Use: "tmux",
31	+	Short: "Utility to get info from tmux",
32	+	Long: core.Banner(),
33	+	RunE: runTmux,
34	+	}
35	+
36	+	tmuxCmd.Flags().BoolVarP(&options.Tmux.ApplyAll, "all", "A", false, "Apply for all tmux sessions")
37	+	tmuxCmd.Flags().StringVarP(&options.Tmux.SelectedWindow, "name", "n", "", "Apply for all tmux sessions")
38	+	tmuxCmd.Flags().StringVarP(&options.Tmux.Exclude, "exclude", "e", "server", "Exclude tmux session")
39	+	tmuxCmd.Flags().IntVarP(&options.Tmux.Limit, "limit", "l", 0, "Size of output content")
40	+
41	+	var cronCmd = &cobra.Command{
42	+	Use: "cron",
43	+	Short: "Utility to run command schedule",
44	+	Long: core.Banner(),
45	+	RunE: runCron,
46	+	}
47	+	cronCmd.Flags().IntVar(&options.Cron.Schedule, "sch", 0, "Number of minutes to schedule the job")
48	+	cronCmd.Flags().BoolVar(&options.Cron.Forever, "for", false, "Keep running forever right after the command done")
49	+	cronCmd.Flags().StringVar(&options.Cron.Command, "cmd", "", "Command to run")
50	+
51	+	// add command
52	+	utilsCmd.AddCommand(cronCmd)
53	+	utilsCmd.AddCommand(tmuxCmd)
54	+	utilsCmd.AddCommand(psCmd)
55	+	utilsCmd.SetHelpFunc(UtilsHelp)
56	+	RootCmd.AddCommand(utilsCmd)
57	+	}
58	+
59	+	func runPs(cmd *cobra.Command, _ []string) error {
60	+	processes, _ := cmd.Flags().GetStringSlice("process")
61	+	if len(processes) == 0 {
62	+	processes = append(processes, libs.BINARY)
63	+	}
64	+
65	+	for _, process := range processes {
66	+	pss := execution.GetOsmProcess(process)
67	+	for _, ps := range pss {
68	+	if options.JsonOutput {
69	+	if data, err := jsoniter.MarshalToString(ps); err == nil {
70	+	fmt.Println(data)
71	+	}
72	+	continue
73	+	}
74	+	fmt.Printf("pid:%v %s %v\n", color.HiCyanString("%v", ps.PID), color.HiMagentaString("--"), ps.Command)
75	+	}
76	+	}
77	+
78	+	return nil
79	+	}
80	+
81	+	func runTmux(_ *cobra.Command, args []string) error {
82	+	tmux, err := core.InitTmux(options)
83	+	if err != nil {
84	+	return err
85	+	}
86	+
87	+	for _, argument := range args {
88	+	switch argument {
89	+	case "l", "ls", "list":
90	+	tmux.ListTmux()
91	+	case "t", "log", "logs", "tai", "tail":
92	+	tmux.CatchSession()
93	+	}
94	+	}
95	+	return nil
96	+	}
97	+
98	+	func runCron(_ *cobra.Command, _ []string) error {
99	+	if options.Cron.Schedule == 0 && options.Cron.Forever == false {
100	+	return fmt.Errorf("missing '--sche' flag")
101	+	}
102	+	if options.Cron.Forever {
103	+	options.Cron.Schedule = -1
104	+	}
105	+	core.RunCron(options.Cron.Command, options.Cron.Schedule)
106	+	return nil
107	+	}
108	+

■ ■ ■ ■ ■ ■

cmd/version.go

1	+	package cmd
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/core"
6	+	"github.com/j3ssie/osmedeus/libs"
7	+	"github.com/j3ssie/osmedeus/utils"
8	+	jsoniter "github.com/json-iterator/go"
9	+	"github.com/mackerelio/go-osstat/cpu"
10	+	"github.com/mackerelio/go-osstat/memory"
11	+	"github.com/spf13/cobra"
12	+	"os"
13	+	"strings"
14	+	"time"
15	+	)
16	+
17	+	func init() {
18	+	var execCmd = &cobra.Command{
19	+	Use: "version",
20	+	Short: "Show core version",
21	+	Long: core.Banner(),
22	+	RunE: runVersion,
23	+	}
24	+	RootCmd.AddCommand(execCmd)
25	+	}
26	+
27	+	func runVersion(_ *cobra.Command, _ []string) error {
28	+	if options.JsonOutput {
29	+	fmt.Println(PrintStat())
30	+	return nil
31	+	}
32	+
33	+	if !options.Verbose {
34	+	fmt.Printf("osmedeus %s by %s\n", libs.VERSION, libs.AUTHOR)
35	+	} else {
36	+	statInfo := PrintStat()
37	+	fmt.Printf("osmedeus %s by %s -- %s\n", libs.VERSION, libs.AUTHOR, statInfo)
38	+	}
39	+	return nil
40	+	}
41	+
42	+	// StatData overview struct
43	+	type StatData struct {
44	+	CPU string `json:"cpu"`
45	+	Mem string `json:"mem"`
46	+	Name string `json:"name"`
47	+	Version string `json:"verion"`
48	+	}
49	+
50	+	// PrintStat print status
51	+	func PrintStat() string {
52	+	data := GetStat()
53	+	if data.CPU.Idle == 0.0 {
54	+	return strings.TrimSpace(utils.Emojif(":thought_balloon:", "not responding"))
55	+	}
56	+	var cpu string
57	+	cpuUsage := 100.0 - data.CPU.Idle
58	+	if cpuUsage <= 20.0 {
59	+	cpu = utils.Emojif(":green_circle:", " cpu: %0.2f", cpuUsage)
60	+	} else if (cpuUsage > 20.0) && (cpuUsage <= 50.0) {
61	+	cpu = utils.Emojif(":green_circle:", " cpu: %0.2f", cpuUsage)
62	+	} else if (cpuUsage > 50.0) && (cpuUsage <= 80.0) {
63	+	cpu = utils.Emojif(":orange_circle:", " cpu: %0.2f", cpuUsage)
64	+	} else {
65	+	cpu = utils.Emojif(":red_circle:", " cpu: %0.2f", cpuUsage)
66	+	}
67	+
68	+	var mem string
69	+	memUsage := 100.0 - (data.Mem.Free/data.Mem.Total)*100
70	+	if memUsage <= 20.0 {
71	+	mem = utils.Emojif(":green_circle:", " mem: %0.2f", memUsage)
72	+	} else if (memUsage > 20.0) && (memUsage <= 50.0) {
73	+	mem = utils.Emojif(":green_circle:", " mem: %0.2f", memUsage)
74	+	} else if (memUsage > 50.0) && (memUsage <= 80.0) {
75	+	mem = utils.Emojif(":orange_circle:", " mem: %0.2f", memUsage)
76	+	} else {
77	+	mem = utils.Emojif(":red_circle:", " mem: %0.2f", memUsage)
78	+	}
79	+
80	+	name, _ := os.Hostname()
81	+	if options.JsonOutput {
82	+	stat := StatData{
83	+	CPU: fmt.Sprintf("%v", cpuUsage),
84	+	Mem: fmt.Sprintf("%v", memUsage),
85	+	Name: name,
86	+	Version: fmt.Sprintf("osmedeus %s by %s", libs.VERSION, libs.AUTHOR),
87	+	}
88	+	if data, err := jsoniter.MarshalToString(stat); err == nil {
89	+	return data
90	+	}
91	+	}
92	+
93	+	return fmt.Sprintf("%s: %12s - %s", name, strings.TrimSpace(cpu), strings.TrimSpace(mem))
94	+	}
95	+
96	+	type ServerStatData struct {
97	+	CPU struct {
98	+	System float64
99	+	User float64
100	+	Idle float64
101	+	}
102	+	Mem struct {
103	+	Total float64
104	+	Used float64
105	+	Free float64
106	+	Cached float64
107	+	}
108	+	}
109	+
110	+	// GetStat get stat data
111	+	func GetStat() ServerStatData {
112	+	var stat ServerStatData
113	+
114	+	before, err := cpu.Get()
115	+	if err != nil {
116	+	return stat
117	+	}
118	+	time.Sleep(time.Duration(1) * time.Second)
119	+	after, err := cpu.Get()
120	+	if err != nil {
121	+	return stat
122	+	}
123	+	total := float64(after.Total - before.Total)
124	+	stat.CPU.User = float64(after.User-before.User) / total * 100
125	+	stat.CPU.System = float64(after.System-before.System) / total * 100
126	+	stat.CPU.Idle = float64(after.Idle-before.Idle) / total * 100
127	+	// memory part
128	+	memory, err := memory.Get()
129	+	if err != nil {
130	+	return stat
131	+	}
132	+	stat.Mem.Total = float64(memory.Total+memory.SwapTotal) / (1024 * 1024 * 1024)
133	+	stat.Mem.Used = float64(memory.Used+memory.SwapUsed) / (1024 * 1024 * 1024)
134	+	stat.Mem.Used = float64(memory.Used+memory.SwapUsed) / (1024 * 1024 * 1024)
135	+	stat.Mem.Cached = float64(memory.Cached) / (1024 * 1024 * 1024)
136	+	stat.Mem.Free = float64(memory.Free+memory.SwapFree) / (1024 * 1024 * 1024)
137	+	return stat
138	+	}
139	+

■ ■ ■ ■ ■ ■

core/backup.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/libs"
6	+	"github.com/j3ssie/osmedeus/utils"
7	+	"os"
8	+	"path"
9	+	)
10	+
11	+	func BackupWorkspace(options libs.Options) {
12	+	if !options.EnableBackup {
13	+	return
14	+	}
15	+
16	+	outputDir := options.Scan.ROptions["Output"]
17	+	dest := path.Join(options.Env.BackupFolder, options.Scan.ROptions["Workspace"]) + ".zip"
18	+	if utils.FileExists(dest) {
19	+	os.Remove(dest)
20	+	}
21	+
22	+	zipCommand := fmt.Sprintf("zip -9 -q -r %s %s", dest, outputDir)
23	+	utils.RunCmdWithOutput(zipCommand)
24	+	if utils.FileExists(dest) {
25	+	utils.GoodF("Backup workspace save at: %s", dest)
26	+	}
27	+	}
28	+

■ ■ ■ ■ ■ ■

core/banner.go

1	+	package core
2	+
3	+	import (
4	+	"github.com/fatih/color"
5	+	"github.com/j3ssie/osmedeus/libs"
6	+	)
7	+
8	+	// Banner print ascii banner
9	+	func Banner() string {
10	+	version := color.HiWhiteString(libs.VERSION)
11	+	author := color.MagentaString(libs.AUTHOR)
12	+	//W := color.HiWhiteString(``)
13	+	b := color.GreenString(``)
14	+
15	+	b += color.GreenString(`
16	+
17	+	.;1tfLCL1,
18	+	.,,..;i;f0G;
19	+	,:,tCC. ...
20	+	;i:fCL,1LLtf1i;,
21	+	.,::tCL1LC1::;, .,,
22	+	;1:tCL,tLt,1:
23	+	,::tLf, 1Lf;::.
24	+	.ii:tLt. .1Lf;i1.
25	+	,:;tf1 1ft;::
26	+	.1;:tf1 `) + color.HiWhiteString(` ,i1t1, `) + color.GreenString(` ift;;1,
27	+	,i:t;f.`) + color.HiWhiteString(` ,LLffLL: `) + color.GreenString(` tft;i:
28	+	.;:;fff `) + color.HiWhiteString(` .LCLLLf,`) + color.GreenString(` 1ffi:;.
29	+	:fi;Lff1. `) + color.HiWhiteString(`,;;:`) + color.GreenString(` ifffi;f;
30	+	.:::tCLLfi:,,:ifLfLt::;.
31	+	,11:1CCCCCLLLLLLf1;1t:
32	+	.it;:;1fLLLLfft1;:;ti.
33	+	,:;::;;;;;;;;;;,
34	+	.,::::::::,.
35	+	`)
36	+
37	+	//
38	+	//
39	+	//b += "\n\t" + color.GreenString(` @@@@@@`)
40	+	//b += "\n\t" + color.GreenString(` .@@' '@@.`)
41	+	//b += "\n\t" + color.GreenString(` :@ @:`)
42	+	//b += "\n\t" + color.GreenString(` :@ %v:@`, W) + color.GreenString(` @:`)
43	+	//b += "\n\t" + color.GreenString(` :@ %v:@`, W) + color.GreenString(` @:`)
44	+	//
45	+	//b += "\n\t" + color.GreenString(` :@ @:`)
46	+	//b += "\n\t" + color.GreenString(` '@@. .@@'`)
47	+	//b += "\n\t" + color.GreenString(` @@@@@@`)
48	+	//b += "\n\t" + color.GreenString(` @@`)
49	+	//b += "\n\t" + color.HiCyanString(` @ `) + color.GreenString(`@@`) + color.HiCyanString(` @`)
50	+	//b += "\n\t" + color.HiWhiteString(` +@@`) + color.GreenString(` @@ `) + color.HiWhiteString(` @@+`)
51	+	//b += "\n\t" + color.GreenString(` @@:@#@,@@,@#@:@@`)
52	+	//b += "\n\t" + color.GreenString(` ;@+@@'#@@@@#'@@+@;`)
53	+	//b += "\n\t" + color.GreenString(` @+ #@@ @@ @@# +@`)
54	+	//b += "\n\t" + color.GreenString(` @@ @+'@@@@@@'+@ @@`)
55	+	//b += "\n\t" + color.GreenString(` @. @ ;@@; @ .@`)
56	+	//b += "\n\t" + color.BlueString(` #@ '@ @; @#`)
57	+
58	+	b += "\n\n\t" + color.GreenString(` Osmedeus Next Generation %v`, version) + color.GreenString(` by %v`, author)
59	+	b += "\n\n" + color.HiCyanString(` %s`, libs.DESC) + "\n"
60	+	b += "\n" + color.HiWhiteString(` ¯\_(ツ)_/¯`) + "\n\n"
61	+	color.Unset()
62	+	return b
63	+	}
64	+

■ ■ ■ ■ ■ ■

core/config.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"net/url"
6	+	"os"
7	+	"path"
8	+	"path/filepath"
9	+	"strings"
10	+
11	+	"github.com/mitchellh/go-homedir"
12	+
13	+	"github.com/j3ssie/osmedeus/execution"
14	+	"github.com/j3ssie/osmedeus/libs"
15	+	"github.com/j3ssie/osmedeus/utils"
16	+
17	+	"github.com/spf13/viper"
18	+	)
19	+
20	+	// InitConfig Init the config
21	+	func InitConfig(options *libs.Options) {
22	+	RootFolder := filepath.Dir(utils.NormalizePath(options.ConfigFile))
23	+	if !utils.FolderExists(RootFolder) {
24	+	os.MkdirAll(RootFolder, 0750)
25	+	}
26	+
27	+	// Base folder
28	+	BaseFolder := utils.NormalizePath(options.Env.BaseFolder)
29	+	if !utils.FolderExists(BaseFolder) {
30	+	os.MkdirAll(BaseFolder, 0750)
31	+	}
32	+
33	+	// init config
34	+	v := viper.New()
35	+	v.AddConfigPath(RootFolder)
36	+	v.SetConfigName("config")
37	+	v.SetConfigType("yaml")
38	+
39	+	if !utils.FileExists(options.ConfigFile) {
40	+	// Some default config if config file doesn't exist
41	+	secret := utils.GenHash(utils.RandomString(8) + utils.GetTS())
42	+	prefix := secret[len(secret)-20 : len(secret)-1]
43	+
44	+	v.SetDefault("Server", map[string]string{
45	+	"bind": "0.0.0.0:8000",
46	+	"cors": "*",
47	+	"secret": secret,
48	+	"prefix": prefix,
49	+	"ui": path.Join(RootFolder, "server/ui"),
50	+	"cert_file": path.Join(RootFolder, "server/ssl/cert.pem"),
51	+	"key_file": path.Join(RootFolder, "server/ssl/key.pem"),
52	+	"master_pass": "",
53	+	})
54	+
55	+	// DB connection config
56	+	dbPath := utils.NormalizePath(path.Join(RootFolder, "sqlite.db"))
57	+	v.SetDefault("Database", map[string]string{
58	+	"db_host": utils.GetOSEnv("DB_HOST", "127.0.0.1"),
59	+	"db_port": utils.GetOSEnv("DB_PORT", "3306"),
60	+	"db_name": utils.GetOSEnv("DB_NAME", "osm-core"),
61	+	"db_user": utils.GetOSEnv("DB_USER", "root"),
62	+	"db_pass": utils.GetOSEnv("DB_PASS", ""),
63	+	// sqlite or mysql
64	+	"db_path": utils.GetOSEnv("DB_PATH", dbPath),
65	+	"db_type": utils.GetOSEnv("DB_TYPE", "sqlite"),
66	+	})
67	+
68	+	// default user
69	+	password := utils.GenHash(utils.GetTS())[:15]
70	+	v.SetDefault("Client", map[string]string{
71	+	"username": "osmedeus",
72	+	"password": password,
73	+	"jwt": "",
74	+	"dest": "http://127.0.0.1:8000",
75	+	})
76	+
77	+	v.SetDefault("Environments", map[string]string{
78	+	// RootFolder --> ~/.osmedeus/
79	+	"storages": path.Join(RootFolder, "storages"),
80	+	"workspaces": path.Join(RootFolder, "workspaces"),
81	+	"backups": path.Join(RootFolder, "backups"),
82	+	"cloud_data": path.Join(RootFolder, "clouds"),
83	+	"provider_config": path.Join(RootFolder, "provider"),
84	+
85	+	// this update casually
86	+	// BaseFolder --> ~/osmedeus-base/
87	+	"workflows": path.Join(BaseFolder, "workflow"),
88	+	"binaries": path.Join(BaseFolder, "binaries"),
89	+	"data": path.Join(BaseFolder, "data"),
90	+	"cloud_config": path.Join(BaseFolder, "cloud"),
91	+	})
92	+
93	+	// things should be reloaded by env
94	+	v.SetDefault("Storages", map[string]string{
95	+	// path of secret key for push result
96	+	// the repo format should be like this "[email protected]:j3ssie/example.git",
97	+	"secret_key": utils.GetOSEnv("SECRET_KEY", "SECRET_KEY"),
98	+	"summary_storage": path.Join(options.Env.RootFolder, "storages/summary"),
99	+	"summary_repo": utils.GetOSEnv("SUMMARY_REPO", "SUMMARY_REPO"),
100	+	"subdomain_storage": path.Join(options.Env.RootFolder, "storages/subdomain"),
101	+	"subdomain_repo": utils.GetOSEnv("SUBDOMAIN_REPO", "SUBDOMAIN_REPO"),
102	+	"assets_storage": path.Join(options.Env.RootFolder, "storages/assets"),
103	+	"assets_repo": utils.GetOSEnv("ASSETS_REPO", "ASSETS_REPO"),
104	+	"ports_storage": path.Join(options.Env.RootFolder, "storages/ports"),
105	+	"ports_repo": utils.GetOSEnv("PORTS_REPO", "PORTS_REPO"),
106	+	"http_storage": path.Join(options.Env.RootFolder, "storages/http"),
107	+	"http_repo": utils.GetOSEnv("HTTP_REPO", "HTTP_REPO"),
108	+	"vuln_storage": path.Join(options.Env.RootFolder, "storages/vuln"),
109	+	"vuln_repo": utils.GetOSEnv("VULN_REPO", "VULN_REPO"),
110	+	"paths_storage": path.Join(options.Env.RootFolder, "storages/paths"),
111	+	"paths_repo": utils.GetOSEnv("PATHS_REPO", "PATHS_REPO"),
112	+	"mics_storage": path.Join(options.Env.RootFolder, "storages/mics"),
113	+	"mics_repo": utils.GetOSEnv("MICS_REPO", "MICS_REPO"),
114	+	})
115	+
116	+	v.SetDefault("Tokens", map[string]string{
117	+	"slack": utils.GetOSEnv("SLACK_API_TOKEN", "SLACK_API_TOKEN"),
118	+	"telegram": utils.GetOSEnv("TELEGRAM_API_TOKEN", "TELEGRAM_API_TOKEN"),
119	+	"gitlab": utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"),
120	+	"github": utils.GetOSEnv("GITHUB_API_KEY", "GITHUB_API_KEY"),
121	+	"discord": utils.GetOSEnv("DISCORD_API_TOKEN", "DISCORD_API_TOKEN"),
122	+	})
123	+
124	+	// dedicated storages
125	+	v.SetDefault("Git", map[string]string{
126	+	"base_url": utils.GetOSEnv("GITLAB_BASE_URL", "https://gitlab.com"),
127	+	"api": utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"),
128	+	"username": utils.GetOSEnv("GITLAB_USER", "GITLAB_USER"),
129	+	"password": utils.GetOSEnv("GITLAB_PASS", "GITLAB_PASS"),
130	+	"group": utils.GetOSEnv("GITLAB_GROUP", "GITLAB_GROUP"),
131	+	"prefix_name": utils.GetOSEnv("GITLAB_PREFIX_NAME", "deosm"),
132	+	"default_tag": utils.GetOSEnv("GITLAB_DEFAULT_TAG", "osmd"),
133	+	"default_user": utils.GetOSEnv("GITLAB_DEFAULT_USER", "j3ssie"),
134	+	"default_uid": utils.GetOSEnv("GITLAB_DEFAULT_UID", "3537075"),
135	+	"destorage": path.Join(options.Env.RootFolder, "destorage"),
136	+	})
137	+
138	+	v.Set("Notification", map[string]string{
139	+	"client_name": utils.GetOSEnv("CLIENT_NAME", "CLIENT_NAME"),
140	+	"slack_status_channel": utils.GetOSEnv("SLACK_STATUS_CHANNEL", "SLACK_STATUS_CHANNEL"),
141	+	"slack_report_channel": utils.GetOSEnv("SLACK_REPORT_CHANNEL", "SLACK_REPORT_CHANNEL"),
142	+	"slack_diff_channel": utils.GetOSEnv("SLACK_DIFF_CHANNEL", "SLACK_DIFF_CHANNEL"),
143	+	"slack_webhook": utils.GetOSEnv("SLACK_WEBHOOK", "SLACK_WEBHOOK"),
144	+	"discord_channel": utils.GetOSEnv("DISCORD_CHANNEL", "DISCORD_CHANNEL"),
145	+	"telegram_channel": utils.GetOSEnv("TELEGRAM_CHANNEL", "TELEGRAM_CHANNEL"),
146	+	"telegram_status_channel": utils.GetOSEnv("TELEGRAM_STATUS_CHANNEL", "TELEGRAM_STATUS_CHANNEL"),
147	+	"telegram_report_channel": utils.GetOSEnv("TELEGRAM_REPORT_CHANNEL", "TELEGRAM_REPORT_CHANNEL"),
148	+	"telegram_sensitive_channel": utils.GetOSEnv("TELEGRAM_SENSITIVE_CHANNEL", "TELEGRAM_SENSITIVE_CHANNEL"),
149	+	"telegram_dirb_channel": utils.GetOSEnv("TELEGRAM_DIRB_CHANNEL", "TELEGRAM_DIRB_CHANNEL"),
150	+	"telegram_mics_channel": utils.GetOSEnv("TELEGRAM_MICS_CHANNEL", "TELEGRAM_MICS_CHANNEL"),
151	+	})
152	+
153	+	// used for scaling
154	+	//v.SetDefault("Master", map[string]string{
155	+	// "host": utils.GetOSEnv("MASTER_HOST", "MASTER_HOST"),
156	+	// "cred": utils.GetOSEnv("MASTER_CRED", "MASTER_CRED"),
157	+	//})
158	+	//v.SetDefault("Pool", map[string]string{
159	+	// "host": utils.GetOSEnv("POOL_HOST", "POOL_HOST"),
160	+	// "cred": utils.GetOSEnv("POOL_CRED", "POOL_CRED"),
161	+	//})
162	+	// enable sync
163	+	//v.SetDefault("Sync", map[string]string{
164	+	// "firebase_url": utils.GetOSEnv("FIREBASE_URL", "FIREBASE_URL"),
165	+	// "firebase_prefix": utils.GetOSEnv("FIREBASE_PREFIX", "FIREBASE_PREFIX"),
166	+	// "firebase_pool": utils.GetOSEnv("FIREBASE_POOL", "FIREBASE_POOL"),
167	+	//})
168	+
169	+	v.SetDefault("Cdn", map[string]string{
170	+	"osm_cdn_url": utils.GetOSEnv("OSM_CDN_URL", "OSM_CDN_URL"),
171	+	"osm_cdn_wsurl": utils.GetOSEnv("OSM_CDN_WSURL", "OSM_CDN_WSURL"),
172	+	"osm_cdn_auth": utils.GetOSEnv("OSM_CDN_AUTH", "OSM_CDN_AUTH"),
173	+	"osm_cdn_prefix": utils.GetOSEnv("OSM_CDN_PREFIX", "OSM_CDN_PREFIX"),
174	+	"osm_cdn_index": utils.GetOSEnv("OSM_CDN_INDEX", "OSM_CDN_INDEX"),
175	+	"osm_cdn_secret": utils.GetOSEnv("OSM_CDN_SECRET", "OSM_CDN_SECRET"),
176	+	})
177	+
178	+	v.SetDefault("Cloud", map[string]string{
179	+	"cloud_public_key": utils.GetOSEnv("CLOUD_PUBLIC_KEY", "CLOUD_PUBLIC_KEY"),
180	+	"cloud_secret_key": utils.GetOSEnv("CLOUD_SECRET_KEY", "CLOUD_SECRET_KEY"),
181	+	"build_repo": utils.GetOSEnv("CLOUD_BUILD_REPO", "CLOUD_BUILD_REPO"),
182	+	})
183	+
184	+	v.SetDefault("Update", map[string]string{
185	+	"update_type": "git",
186	+	"update_url": utils.GetOSEnv("UPDATE_BASE_URL", "UPDATE_BASE_URL"),
187	+	"update_date": utils.GetOSEnv("UPDATE_DATE", "UPDATE_DATE"),
188	+	"update_meta": utils.GetOSEnv("META_URL", "META_URL"),
189	+	"workflow": utils.GetOSEnv("UPDATE_URL", "UPDATE_URL"),
190	+	})
191	+
192	+	v.SetDefault("Mics", map[string]string{
193	+	"docs": utils.GetOSEnv("OSM_DOCS", libs.DOCS),
194	+	})
195	+
196	+	v.WriteConfigAs(options.ConfigFile)
197	+	}
198	+
199	+	GetEnv(options)
200	+	GetStorages(options)
201	+	GetNotification(options)
202	+	GetServer(options)
203	+	GetClient(options)
204	+	GetRemote(options)
205	+	GetGit(options)
206	+	//GetSync(options)
207	+	GetCdn(options)
208	+	SetupOpt(options)
209	+
210	+	}
211	+
212	+	// LoadConfig load config
213	+	func LoadConfig(options libs.Options) (*viper.Viper, error) {
214	+	options.ConfigFile, _ = homedir.Expand(options.ConfigFile)
215	+	RootFolder := filepath.Dir(options.ConfigFile)
216	+	v := viper.New()
217	+	v.SetConfigName("config")
218	+	v.SetConfigType("yaml")
219	+	v.AddConfigPath(RootFolder)
220	+	// InitConfig(&options)
221	+	if err := v.ReadInConfig(); err != nil {
222	+	InitConfig(&options)
223	+	return v, nil
224	+	}
225	+	return v, nil
226	+	}
227	+
228	+	// GetEnv get environment options
229	+	func GetEnv(options *libs.Options) {
230	+	v, _ := LoadConfig(*options)
231	+	envs := v.GetStringMapString("Environments")
232	+
233	+	// config
234	+	options.Env.BinariesFolder = utils.NormalizePath(envs["binaries"])
235	+	utils.MakeDir(options.Env.BinariesFolder)
236	+
237	+	options.Env.DataFolder = utils.NormalizePath(envs["data"])
238	+	utils.MakeDir(options.Env.DataFolder)
239	+	// ose folder
240	+	options.Env.OseFolder = path.Join(options.Env.BaseFolder, "ose")
241	+	utils.MakeDir(options.Env.DataFolder)
242	+
243	+	options.Env.ScriptsFolder = path.Join(options.Env.BaseFolder, "scripts")
244	+	options.Env.UIFolder = path.Join(options.Env.BaseFolder, "ui")
245	+
246	+	// local data
247	+	options.Env.StoragesFolder = utils.NormalizePath(envs["storages"])
248	+	utils.MakeDir(options.Env.StoragesFolder)
249	+	options.Env.WorkspacesFolder = utils.NormalizePath(envs["workspaces"])
250	+	utils.MakeDir(options.Env.WorkspacesFolder)
251	+	options.Env.WorkFlowsFolder = utils.NormalizePath(envs["workflows"])
252	+
253	+	if utils.FileExists(path.Join(options.Env.WorkFlowsFolder, "premium.md")) {
254	+	options.PremiumPackage = true
255	+	}
256	+
257	+	// backup data
258	+	options.Env.BackupFolder = utils.NormalizePath(envs["backups"])
259	+	utils.MakeDir(options.Env.BackupFolder)
260	+
261	+	// cloud stuff
262	+	options.Env.ProviderFolder = utils.NormalizePath(envs["provider_config"])
263	+	utils.MakeDir(options.Env.ProviderFolder)
264	+	options.Env.CloudDataFolder = utils.NormalizePath(envs["cloud_data"])
265	+	utils.MakeDir(options.Env.CloudDataFolder)
266	+	options.Env.CloudConfigFolder = utils.NormalizePath(envs["cloud_config"])
267	+
268	+	cloud := v.GetStringMapString("Cloud")
269	+	options.Cloud.BuildRepo = cloud["build_repo"]
270	+
271	+	// load the config file here
272	+	options.Cloud.SecretKey = utils.NormalizePath(cloud["cloud_secret_key"])
273	+	options.Cloud.PublicKey = utils.NormalizePath(cloud["cloud_public_key"])
274	+	if utils.FileExists(options.Cloud.SecretKey) {
275	+	options.Cloud.SecretKeyContent = strings.TrimSpace(utils.GetFileContent(options.Cloud.SecretKey))
276	+	options.Cloud.PublicKeyContent = strings.TrimSpace(utils.GetFileContent(options.Cloud.PublicKey))
277	+	}
278	+
279	+	//
280	+	//options.Env.RootFolder = utils.NormalizePath(options.Env.RootFolder)
281	+	//options.Env.BaseFolder = utils.NormalizePath(options.Env.BaseFolder)
282	+
283	+	update := v.GetStringMapString("Update")
284	+	options.Update.UpdateURL = update["update_url"]
285	+	options.Update.UpdateType = update["update_type"]
286	+	options.Update.UpdateDate = update["update_date"]
287	+	options.Update.UpdateKey = update["update_key"]
288	+	options.Update.MetaDataURL = update["update_meta"]
289	+	UpdateMetadata(*options)
290	+	}
291	+
292	+	// SetupOpt get storage repos
293	+	func SetupOpt(options *libs.Options) {
294	+	// auto append PATH with Plugin folder
295	+	osPATH := utils.GetOSEnv("PATH", "PATH")
296	+	if !strings.Contains(osPATH, options.Env.BinariesFolder) {
297	+	utils.DebugF("Append $PATH with: %s", options.Env.BinariesFolder)
298	+	os.Setenv("PATH", fmt.Sprintf("%s:%s", osPATH, options.Env.BinariesFolder))
299	+	}
300	+
301	+	/* some special conditions below */
302	+
303	+	// change {{.Storage}} from ~/.osmedeus/storages to ~/.osmedeus/destorages
304	+	if options.EnableDeStorage {
305	+	utils.DebugF("Dedicated Storage Enabled")
306	+	options.Env.StoragesFolder = options.Git.DeStorage
307	+	if !utils.FolderExists(options.Env.StoragesFolder) {
308	+	utils.MakeDir(options.Env.StoragesFolder)
309	+	}
310	+	}
311	+	}
312	+
313	+	func GetStorages(options *libs.Options) {
314	+	v, _ := LoadConfig(*options)
315	+	storages := v.GetStringMapString("Storages")
316	+
317	+	for k, v := range storages {
318	+	storages[k], _ = homedir.Expand(v)
319	+	}
320	+	storagesOptions := make(map[string]string)
321	+	// cloning stuff
322	+	// path to private to push result
323	+	storagesOptions["secret_key"] = storages["secret_key"]
324	+	options.Storages = storagesOptions
325	+
326	+	// load default existing key if it exists
327	+	if options.Storages["secret_key"] == "" \|\| options.Storages["secret_key"] == "SECRET_KEY" {
328	+	// ~/.osmedeus/secret_key.private
329	+	secretKey := path.Join(options.Env.RootFolder, "secret_key.private")
330	+	if utils.FileExists(secretKey) {
331	+	options.Storages["secret_key"] = secretKey
332	+	} else {
333	+	secretKey = path.Join(options.Env.BaseFolder, "secret/secret_key.private")
334	+	if utils.FileExists(secretKey) {
335	+	options.Storages["secret_key"] = secretKey
336	+	}
337	+	}
338	+	}
339	+
340	+	// disable git feature or no secret key found
341	+	if options.NoGit {
342	+	options.Storages["secret_key"] = ""
343	+	}
344	+	if options.Storages["secret_key"] == "" {
345	+	options.NoGit = true
346	+	}
347	+
348	+	if options.CustomGit {
349	+	// in case custom repo is set
350	+	for _, env := range os.Environ() {
351	+	// the ENV should be OSM_SUMMARY_STORAGE, OSM_SUMMARY_REPO
352	+	if strings.HasSuffix(env, "OSM_") {
353	+	data := strings.Split(env, "=")
354	+	key := strings.ToLower(data[0])
355	+	value := strings.Replace(env, data[0]+"=", "", -1)
356	+
357	+	if strings.HasSuffix(key, "summary_storage") {
358	+	storagesOptions["summary_storage"] = value
359	+	}
360	+	if strings.HasSuffix(key, "summary_repo") {
361	+	storagesOptions["summary_repo"] = value
362	+	}
363	+
364	+	if strings.HasSuffix(key, "assets_storage") {
365	+	storagesOptions["assets_storage"] = value
366	+	}
367	+	if strings.HasSuffix(key, "assets_repo") {
368	+	storagesOptions["assets_repo"] = value
369	+	}
370	+
371	+	if strings.HasSuffix(key, "ports_storage") {
372	+	storagesOptions["ports_storage"] = value
373	+	}
374	+	if strings.HasSuffix(key, "ports_repo") {
375	+	storagesOptions["ports_repo"] = value
376	+	}
377	+	}
378	+	}
379	+	}
380	+
381	+	storagesOptions[storages["summary_storage"]] = storages["summary_repo"]
382	+	storagesOptions[storages["subdomain_storage"]] = storages["subdomain_repo"]
383	+	storagesOptions[storages["http_storage"]] = storages["http_repo"]
384	+	storagesOptions[storages["assets_storage"]] = storages["assets_repo"]
385	+	storagesOptions[storages["mics_storage"]] = storages["mics_repo"]
386	+	storagesOptions[storages["ports_storage"]] = storages["ports_repo"]
387	+	storagesOptions[storages["paths_storage"]] = storages["paths_repo"]
388	+	storagesOptions[storages["vuln_storage"]] = storages["vuln_repo"]
389	+
390	+	options.Storages = storagesOptions
391	+	if options.NoGit {
392	+	return
393	+	}
394	+
395	+	execution.CloneRepo(storages["summary_repo"], storages["summary_storage"], *options)
396	+	execution.CloneRepo(storages["http_repo"], storages["http_storage"], *options)
397	+	execution.CloneRepo(storages["assets_repo"], storages["assets_storage"], *options)
398	+	execution.CloneRepo(storages["subdomain_repo"], storages["subdomain_storage"], *options)
399	+	execution.CloneRepo(storages["ports_repo"], storages["ports_storage"], *options)
400	+	execution.CloneRepo(storages["mics_repo"], storages["mics_storage"], *options)
401	+	execution.CloneRepo(storages["paths_repo"], storages["paths_storage"], *options)
402	+	execution.CloneRepo(storages["vuln_repo"], storages["vuln_storage"], *options)
403	+	}
404	+
405	+	// GetNotification get storge repos
406	+	func GetNotification(options *libs.Options) {
407	+	v, _ := LoadConfig(*options)
408	+	noti := v.GetStringMapString("Notification")
409	+	tokens := v.GetStringMapString("Tokens")
410	+
411	+	options.Noti.SlackToken = tokens["slack"]
412	+	options.Noti.TelegramToken = tokens["telegram"]
413	+	options.Noti.DiscordToken = tokens["discord"]
414	+	options.Noti.ClientName = noti["client_name"]
415	+
416	+	options.Noti.SlackStatusChannel = noti["slack_status_channel"]
417	+	options.Noti.SlackReportChannel = noti["slack_report_channel"]
418	+	options.Noti.SlackDiffChannel = noti["slack_diff_channel"]
419	+	options.Noti.SlackWebHook = noti["slack_webhook"]
420	+	options.Noti.TelegramChannel = noti["telegram_channel"]
421	+	options.Noti.TelegramSensitiveChannel = noti["telegram_sensitive_channel"]
422	+	options.Noti.TelegramReportChannel = noti["telegram_report_channel"]
423	+	options.Noti.TelegramStatusChannel = noti["telegram_status_channel"]
424	+	options.Noti.TelegramDirbChannel = noti["telegram_dirb_channel"]
425	+	options.Noti.TelegramMicsChannel = noti["telegram_mics_channel"]
426	+	}
427	+
428	+	// GetServer get server options
429	+	func GetServer(options *libs.Options) {
430	+	v, _ := LoadConfig(*options)
431	+	server := v.GetStringMapString("Server")
432	+
433	+	options.Server.Bind = server["bind"]
434	+	options.Server.Cors = server["cors"]
435	+	options.Server.JWTSecret = server["secret"]
436	+	options.Server.StaticPrefix = server["prefix"]
437	+	options.Server.UIPath = utils.NormalizePath(server["ui"])
438	+	utils.MakeDir(path.Dir(options.Server.UIPath))
439	+
440	+	options.Server.MasterPassword = server["master_pass"]
441	+	options.Server.CertFile = utils.NormalizePath(server["cert_file"])
442	+	options.Server.KeyFile = utils.NormalizePath(server["key_file"])
443	+	utils.MakeDir(path.Dir(options.Server.CertFile))
444	+
445	+	db := v.GetStringMapString("Database")
446	+
447	+	options.Server.DBPath = utils.NormalizePath(db["db_path"])
448	+	options.Server.DBType = db["db_type"]
449	+	// this should be remote one
450	+	if options.Server.DBType == "mysql" {
451	+	options.Server.DBUser = db["db_user"]
452	+	options.Server.DBPass = db["db_pass"]
453	+	options.Server.DBHost = db["db_host"]
454	+	options.Server.DBPort = db["db_port"]
455	+	options.Server.DBName = db["db_name"]
456	+
457	+	// â€œuser:password@/dbname?charset=utf8&parseTime=True&loc=Localâ€
458	+	cred := fmt.Sprintf("%v:%v", options.Server.DBUser, options.Server.DBPass)
459	+	dest := fmt.Sprintf("%v:%v", options.Server.DBHost, options.Server.DBPort)
460	+	dbURL := fmt.Sprintf("%v@tcp(%v)/%v?charset=utf8&parseTime=True&loc=Local", cred, dest, options.Server.DBName)
461	+	options.Server.DBConnection = dbURL
462	+	}
463	+	}
464	+
465	+	// GetRemote get server remote options
466	+	func GetRemote(options *libs.Options) {
467	+	v, _ := LoadConfig(*options)
468	+	master := v.GetStringMapString("Master")
469	+	pool := v.GetStringMapString("Pool")
470	+
471	+	options.Remote.MasterHost = master["host"]
472	+	options.Remote.MasterCred = master["cred"]
473	+	options.Remote.PoolHost = pool["host"]
474	+	options.Remote.PoolCred = pool["cred"]
475	+	}
476	+
477	+	// GetClient get options for client
478	+	func GetClient(options *libs.Options) map[string]string {
479	+	v, _ := LoadConfig(*options)
480	+	client := v.GetStringMapString("Client")
481	+	options.Client.Username = client["username"]
482	+	options.Client.Password = client["password"]
483	+	return client
484	+	}
485	+
486	+	// GetGit get options for client
487	+	func GetGit(options *libs.Options) {
488	+	v, _ := LoadConfig(*options)
489	+	git := v.GetStringMapString("Git")
490	+	options.Git.BaseURL = git["base_url"]
491	+	options.Git.DeStorage = git["destorage"]
492	+	options.Git.Token = git["api"]
493	+	options.Git.Username = git["username"]
494	+	options.Git.Password = git["password"]
495	+	options.Git.Group = git["group"]
496	+	options.Git.DefaultPrefix = git["prefix_name"]
497	+	options.Git.DefaultTag = git["default_tag"]
498	+	options.Git.DefaultUser = git["default_user"]
499	+	options.Git.DefaultUID = utils.StrToInt(git["default_uid"])
500	+	}
501	+
502	+	// GetSync get options for client
503	+	func GetSync(options *libs.Options) {
504	+	v, _ := LoadConfig(*options)
505	+	fb := v.GetStringMapString("Sync")
506	+	options.Sync.BaseURL = fb["firebase_url"]
507	+	options.Sync.Prefix = fb["firebase_prefix"]
508	+	options.Sync.Pool = fb["firebase_pool"]
509	+	}
510	+
511	+	// GetCdn get options for client
512	+	func GetCdn(options *libs.Options) {
513	+	v, _ := LoadConfig(*options)
514	+	cdn := v.GetStringMapString("Cdn")
515	+	options.Cdn.URL = cdn["osm_cdn_url"]
516	+	options.Cdn.WSURL = cdn["osm_cdn_wsurl"]
517	+	options.Cdn.Prefix = cdn["osm_cdn_prefix"]
518	+	options.Cdn.Index = cdn["osm_cdn_index"]
519	+
520	+	// in case we have prefix
521	+	if options.Cdn.Prefix != "OSM_CDN_PREFIX" {
522	+	u, err := url.Parse(options.Cdn.URL)
523	+	if err == nil {
524	+	u.Path = path.Join(u.Path, options.Cdn.Prefix)
525	+	options.Cdn.URL = u.String()
526	+	}
527	+
528	+	u, err = url.Parse(options.Cdn.WSURL)
529	+	if err == nil {
530	+	u.Path = path.Join(u.Path, options.Cdn.Prefix)
531	+	options.Cdn.WSURL = u.String()
532	+	}
533	+	}
534	+
535	+	options.Cdn.Auth = cdn["osm_cdn_auth"]
536	+	}
537	+
538	+	// ReloadConfig get credentials
539	+	func ReloadConfig(options libs.Options) {
540	+	utils.InforF("Reload Env for config file: %v", options.ConfigFile)
541	+	v, _ := LoadConfig(options)
542	+
543	+	// options.ConfigFile, _ = homedir.Expand(options.ConfigFile)
544	+	RootFolder := filepath.Dir(utils.NormalizePath(options.ConfigFile))
545	+	if !utils.FolderExists(RootFolder) {
546	+	os.MkdirAll(RootFolder, 0750)
547	+	}
548	+	// Base folder
549	+	BaseFolder := utils.NormalizePath(options.Env.BaseFolder)
550	+	if !utils.FolderExists(BaseFolder) {
551	+	os.MkdirAll(BaseFolder, 0750)
552	+	}
553	+
554	+	v.Set("Environments", map[string]string{
555	+	// RootFolder --> ~/.osmedeus/
556	+	"storages": path.Join(RootFolder, "storages"),
557	+	"workspaces": path.Join(RootFolder, "workspaces"),
558	+	"backups": path.Join(RootFolder, "backups"),
559	+	"cloud_data": path.Join(RootFolder, "clouds"),
560	+	"provider_config": path.Join(RootFolder, "provider"),
561	+
562	+	// this update casually
563	+	// BaseFolder --> ~/osmedeus-base/
564	+	"workflows": path.Join(BaseFolder, "workflow"),
565	+	"binaries": path.Join(BaseFolder, "binaries"),
566	+	"data": path.Join(BaseFolder, "data"),
567	+	"cloud_config": path.Join(BaseFolder, "cloud"),
568	+	})
569	+
570	+	//
571	+	//v.Set("Environments", map[string]string{
572	+	// //"workflows": path.Join(RootFolder, "core/workflow"),
573	+	// "workflows": path.Join(BaseFolder, "core/workflow"),
574	+	// "binaries": path.Join(BaseFolder, "binaries"),
575	+	// "storages": path.Join(RootFolder, "storages"),
576	+	// "workspaces": path.Join(RootFolder, "workspaces"),
577	+	// "plugins": path.Join(BaseFolder, "plugins"),
578	+	// "data": path.Join(BaseFolder, "data"),
579	+	// "cloud_data": path.Join(RootFolder, "clouds"),
580	+	// "provider_config": path.Join(RootFolder, "provider"),
581	+	// "cloud_config": path.Join(BaseFolder, "cloud"),
582	+	// //
583	+	// "backups": path.Join(RootFolder, "backups"),
584	+	//})
585	+
586	+	v.Set("Cloud", map[string]string{
587	+	"cloud_secret_key": utils.GetOSEnv("CLOUD_SECRET_KEY", "CLOUD_SECRET_KEY"),
588	+	"cloud_public_key": utils.GetOSEnv("CLOUD_PUBLIC_KEY", "CLOUD_PUBLIC_KEY"),
589	+	"build_repo": utils.GetOSEnv("CLOUD_BUILD_REPO", "CLOUD_BUILD_REPO"),
590	+	})
591	+
592	+	// things should be reload by env
593	+	v.Set("Storages", map[string]string{
594	+	// path of secret key for push result
595	+	// the repo format should be like this "[email protected]:j3ssie/example.git",
596	+	"secret_key": utils.GetOSEnv("SECRET_KEY", "SECRET_KEY"),
597	+	"summary_storage": path.Join(options.Env.RootFolder, "storages/summary"),
598	+	"summary_repo": utils.GetOSEnv("SUMMARY_REPO", "SUMMARY_REPO"),
599	+	"subdomain_storage": path.Join(options.Env.RootFolder, "storages/subdomain"),
600	+	"subdomain_repo": utils.GetOSEnv("SUBDOMAIN_REPO", "SUBDOMAIN_REPO"),
601	+	"assets_storage": path.Join(options.Env.RootFolder, "storages/assets"),
602	+	"assets_repo": utils.GetOSEnv("ASSETS_REPO", "ASSETS_REPO"),
603	+	"ports_storage": path.Join(options.Env.RootFolder, "storages/ports"),
604	+	"ports_repo": utils.GetOSEnv("PORTS_REPO", "PORTS_REPO"),
605	+	"http_storage": path.Join(options.Env.RootFolder, "storages/http"),
606	+	"http_repo": utils.GetOSEnv("HTTP_REPO", "HTTP_REPO"),
607	+	"vuln_storage": path.Join(options.Env.RootFolder, "storages/vuln"),
608	+	"vuln_repo": utils.GetOSEnv("VULN_REPO", "VULN_REPO"),
609	+	"paths_storage": path.Join(options.Env.RootFolder, "storages/paths"),
610	+	"paths_repo": utils.GetOSEnv("PATHS_REPO", "PATHS_REPO"),
611	+	"mics_storage": path.Join(options.Env.RootFolder, "storages/mics"),
612	+	"mics_repo": utils.GetOSEnv("MICS_REPO", "MICS_REPO"),
613	+	})
614	+
615	+	v.Set("Tokens", map[string]string{
616	+	"slack": utils.GetOSEnv("SLACK_API_TOKEN", "SLACK_API_TOKEN"),
617	+	"gitlab": utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"),
618	+	"github": utils.GetOSEnv("GITHUB_API_KEY", "GITHUB_API_KEY"),
619	+	"discord": utils.GetOSEnv("DISCORD_API_TOKEN", "DISCORD_API_TOKEN"),
620	+	"telegram": utils.GetOSEnv("TELEGRAM_API_TOKEN", "TELEGRAM_API_TOKEN"),
621	+	})
622	+
623	+	v.Set("Git", map[string]string{
624	+	"base_url": utils.GetOSEnv("GITLAB_BASE_URL", "https://gitlab.com"),
625	+	"api": utils.GetOSEnv("GITLAB_API_TOKEN", "GITLAB_API_TOKEN"),
626	+	"username": utils.GetOSEnv("GITLAB_USER", "GITLAB_USER"),
627	+	"password": utils.GetOSEnv("GITLAB_PASS", "GITLAB_PASS"),
628	+	"group": utils.GetOSEnv("GITLAB_GROUP", "GITLAB_GROUP"),
629	+	"prefix_name": utils.GetOSEnv("GITLAB_PREFIX_NAME", "deosm"),
630	+	"default_tag": utils.GetOSEnv("GITLAB_DEFAULT_TAG", "osmd"),
631	+	"default_user": utils.GetOSEnv("GITLAB_DEFAULT_USER", "j3ssie"),
632	+	"default_uid": utils.GetOSEnv("GITLAB_DEFAULT_UID", "3537075"),
633	+	"destorage": path.Join(options.Env.RootFolder, "destorage"),
634	+	})
635	+
636	+	v.Set("Notification", map[string]string{
637	+	"client_name": utils.GetOSEnv("CLIENT_NAME", "CLIENT_NAME"),
638	+	"slack_status_channel": utils.GetOSEnv("SLACK_STATUS_CHANNEL", "SLACK_STATUS_CHANNEL"),
639	+	"slack_report_channel": utils.GetOSEnv("SLACK_REPORT_CHANNEL", "SLACK_REPORT_CHANNEL"),
640	+	"slack_diff_channel": utils.GetOSEnv("SLACK_DIFF_CHANNEL", "SLACK_DIFF_CHANNEL"),
641	+	"slack_webhook": utils.GetOSEnv("SLACK_WEBHOOK", "SLACK_WEBHOOK"),
642	+	"discord_channel": utils.GetOSEnv("DISCORD_CHANNEL", "DISCORD_CHANNEL"),
643	+	"telegram_channel": utils.GetOSEnv("TELEGRAM_CHANNEL", "TELEGRAM_CHANNEL"),
644	+	"telegram_status_channel": utils.GetOSEnv("TELEGRAM_STATUS_CHANNEL", "TELEGRAM_STATUS_CHANNEL"),
645	+	"telegram_report_channel": utils.GetOSEnv("TELEGRAM_REPORT_CHANNEL", "TELEGRAM_REPORT_CHANNEL"),
646	+	"telegram_sensitive_channel": utils.GetOSEnv("TELEGRAM_SENSITIVE_CHANNEL", "TELEGRAM_SENSITIVE_CHANNEL"),
647	+	"telegram_dirb_channel": utils.GetOSEnv("TELEGRAM_DIRB_CHANNEL", "TELEGRAM_DIRB_CHANNEL"),
648	+	"telegram_mics_channel": utils.GetOSEnv("TELEGRAM_MICS_CHANNEL", "TELEGRAM_MICS_CHANNEL"),
649	+	})
650	+
651	+	v.Set("Cdn", map[string]string{
652	+	"osm_cdn_url": utils.GetOSEnv("OSM_CDN_URL", "OSM_CDN_URL"),
653	+	"osm_cdn_wsurl": utils.GetOSEnv("OSM_CDN_WSURL", "OSM_CDN_WSURL"),
654	+	"osm_cdn_auth": utils.GetOSEnv("OSM_CDN_AUTH", "OSM_CDN_AUTH"),
655	+	"osm_cdn_prefix": utils.GetOSEnv("OSM_CDN_PREFIX", "OSM_CDN_PREFIX"),
656	+	"osm_cdn_index": utils.GetOSEnv("OSM_CDN_INDEX", "OSM_CDN_INDEX"),
657	+	"osm_cdn_secret": utils.GetOSEnv("OSM_CDN_SECRET", "OSM_CDN_SECRET"),
658	+	})
659	+
660	+	v.Set("Update", map[string]string{
661	+	"update_url": utils.GetOSEnv("UPDATE_BASE_URL", "UPDATE_BASE_URL"),
662	+	"update_date": utils.GetOSEnv("UPDATE_DATE", "UPDATE_DATE"),
663	+	"update_meta": utils.GetOSEnv("META_URL", "META_URL"),
664	+	"workflow": utils.GetOSEnv("UPDATE_URL", "UPDATE_URL"),
665	+	})
666	+
667	+	//
668	+	//v.Set("Sync", map[string]string{
669	+	// "firebase_url": utils.GetOSEnv("FIREBASE_URL", "FIREBASE_URL"),
670	+	// "firebase_prefix": utils.GetOSEnv("FIREBASE_PREFIX", "FIREBASE_PREFIX"),
671	+	// "firebase_pool": utils.GetOSEnv("FIREBASE_POOL", "FIREBASE_POOL"),
672	+	//})
673	+	//
674	+	//v.Set("Master", map[string]string{
675	+	// "host": utils.GetOSEnv("MASTER_HOST", "MASTER_HOST"),
676	+	// "cred": utils.GetOSEnv("MASTER_CRED", "MASTER_CRED"),
677	+	//})
678	+	//v.Set("Pool", map[string]string{
679	+	// "host": utils.GetOSEnv("POOL_HOST", "POOL_HOST"),
680	+	// "cred": utils.GetOSEnv("POOL_CRED", "POOL_CRED"),
681	+	//})
682	+
683	+	v.WriteConfig()
684	+	}
685	+

■ ■ ■ ■ ■ ■

core/cron.go

1	+	package core
2	+
3	+	import (
4	+	"github.com/fatih/color"
5	+	"github.com/j3ssie/osmedeus/utils"
6	+	"github.com/jasonlvhit/gocron"
7	+	"github.com/spf13/cast"
8	+	)
9	+
10	+	func taskWithParams(cmd string) {
11	+	utils.InforF("Exec: %v", color.HiMagentaString(cmd))
12	+	utils.RunOSCommand(cmd)
13	+	}
14	+
15	+	func RunCron(cmd string, schedule int) {
16	+
17	+	if schedule == -1 {
18	+	utils.InforF("Run command forever: %v", cmd)
19	+	for {
20	+	taskWithParams(cmd)
21	+	}
22	+	}
23	+
24	+	utils.InforF("Start cron job with %v seconds: %v", schedule, color.HiCyanString(cmd))
25	+	gocron.Every(cast.ToUint64(schedule)).Minutes().Do(taskWithParams, cmd)
26	+	<-gocron.Start()
27	+	}
28	+

■ ■ ■ ■ ■ ■

core/db.go

1	+	package core
2	+
3	+	import (
4	+	"github.com/j3ssie/osmedeus/database"
5	+	"github.com/j3ssie/osmedeus/libs"
6	+	"github.com/j3ssie/osmedeus/utils"
7	+	jsoniter "github.com/json-iterator/go"
8	+	"github.com/robertkrimen/otto"
9	+	"github.com/spf13/cast"
10	+	"os"
11	+	"path"
12	+	"strings"
13	+	)
14	+
15	+	func (r *Runner) LoadDBScripts() string {
16	+	var output string
17	+
18	+	r.VM.Set("TotalSubdomain", func(call otto.FunctionCall) otto.Value {
19	+	length := utils.FileLength(call.Argument(0).String())
20	+	r.TargetObj.TotalAssets = length
21	+	return otto.Value{}
22	+	})
23	+
24	+	r.VM.Set("TotalDns", func(call otto.FunctionCall) otto.Value {
25	+	length := utils.FileLength(call.Argument(0).String())
26	+	r.TargetObj.TotalDns = length
27	+	return otto.Value{}
28	+	})
29	+
30	+	r.VM.Set("TotalScreenShot", func(call otto.FunctionCall) otto.Value {
31	+	length := utils.FileLength(call.Argument(0).String())
32	+	r.TargetObj.TotalScreenShot = length
33	+	return otto.Value{}
34	+	})
35	+
36	+	r.VM.Set("TotalTech", func(call otto.FunctionCall) otto.Value {
37	+	length := utils.FileLength(call.Argument(0).String())
38	+	r.TargetObj.TotalTech = length
39	+	return otto.Value{}
40	+	})
41	+
42	+	r.VM.Set("TotalVulnerability", func(call otto.FunctionCall) otto.Value {
43	+	data := utils.ReadingFileUnique(call.Argument(0).String())
44	+	var length int
45	+	for _, line := range data {
46	+	if !strings.Contains(line, "-info") {
47	+	length += 1
48	+	}
49	+	}
50	+	r.TargetObj.TotalVulnerability = length
51	+	return otto.Value{}
52	+	})
53	+
54	+	r.VM.Set("TotalArchive", func(call otto.FunctionCall) otto.Value {
55	+	length := utils.FileLength(call.Argument(0).String())
56	+	r.TargetObj.TotalArchive = length
57	+	return otto.Value{}
58	+	})
59	+
60	+	r.VM.Set("TotalLink", func(call otto.FunctionCall) otto.Value {
61	+	length := utils.FileLength(call.Argument(0).String())
62	+	r.TargetObj.TotalLink = length
63	+	return otto.Value{}
64	+	})
65	+
66	+	r.VM.Set("TotalDirb", func(call otto.FunctionCall) otto.Value {
67	+	length := utils.FileLength(call.Argument(0).String())
68	+	r.TargetObj.TotalDirb = length
69	+	return otto.Value{}
70	+	})
71	+
72	+	// CreateReport('report', 'subdomain')
73	+	r.VM.Set("CreateReport", func(call otto.FunctionCall) otto.Value {
74	+	args := call.ArgumentList
75	+	report := args[0].String()
76	+	if utils.FileExists(report) {
77	+	return otto.Value{}
78	+	}
79	+	moduleName := "inline"
80	+	if len(args) > 1 {
81	+	moduleName = args[1].String()
82	+	}
83	+
84	+	reportObj := database.Report{
85	+	ReportName: path.Base(report),
86	+	Module: moduleName,
87	+	ReportPath: report,
88	+	ReportType: "",
89	+	TargetRefer: r.TargetObj.ID,
90	+	}
91	+	database.NewReport(&reportObj)
92	+	return otto.Value{}
93	+	})
94	+
95	+	return output
96	+	}
97	+
98	+	func (r *Runner) DBNewTarget() {
99	+	if r.Opt.NoDB {
100	+	return
101	+	}
102	+
103	+	//// this is just sample org
104	+	//org := database.Org{
105	+	// Name: "Sample Org",
106	+	// Desc: "Sample Desc",
107	+	// //Targets: nil,
108	+	//}
109	+	//database.NewOrg(&org)
110	+
111	+	r.TargetObj = database.Target{
112	+	InputName: r.Input,
113	+	Workspace: r.Workspace,
114	+	InputType: r.InputType,
115	+	//OrgRefer: org.ID,
116	+	}
117	+	database.DBUpdateTarget(&r.TargetObj)
118	+	}
119	+
120	+	func (r *Runner) DBNewScan() {
121	+	if r.Opt.NoDB {
122	+	return
123	+	}
124	+
125	+	r.ScanObj = database.Scan{
126	+	TaskType: r.RoutineType,
127	+	TaskName: path.Base(r.RoutineName),
128	+
129	+	// this should be user id as uuid
130	+	//UID: r.RunnerSource,
131	+
132	+	TotalSteps: r.TotalSteps,
133	+	InputName: r.Input,
134	+	InputType: r.InputType,
135	+
136	+	LogFile: r.Opt.LogFile,
137	+	Target: r.TargetObj,
138	+	ProcessID: os.Getpid(),
139	+	IsRunning: true,
140	+	IsDone: false,
141	+	IsPrepared: true,
142	+	}
143	+
144	+	if r.Opt.ScanID != "" {
145	+	utils.InforF("Continue scanning on scan id: %v", r.Opt.ScanID)
146	+	r.ScanObj.ID = cast.ToUint(r.Opt.ScanID)
147	+	}
148	+
149	+	if r.RunnerType == "cloud" {
150	+	r.ScanObj.IsCloud = true
151	+	}
152	+
153	+	database.DBNewScan(&r.ScanObj)
154	+	}
155	+
156	+	func (r *Runner) DBUpdateScan() {
157	+	r.ScanObj.DoneStep = r.DoneStep
158	+	r.ScanObj.CurrentModule = r.CurrentModule
159	+	r.ScanObj.RunningTime = r.RunningTime
160	+	r.ScanObj.ProcessID = os.Getpid()
161	+
162	+	if r.ScanObj.DoneStep == r.ScanObj.TotalSteps {
163	+	r.ScanObj.IsDone = true
164	+	r.ScanObj.IsRunning = false
165	+	} else {
166	+	r.ScanObj.IsRunning = true
167	+	r.ScanObj.IsDone = false
168	+	}
169	+
170	+	utils.DebugF("[DB] Done module %v with %v/%v steps ", r.CurrentModule, r.DoneStep, r.TotalSteps)
171	+	database.DBUpdateScan(&r.ScanObj)
172	+
173	+	r.ScanObj.Target = r.TargetObj
174	+	runtimeData, err := jsoniter.MarshalToString(r.ScanObj)
175	+	if err == nil {
176	+	utils.WriteToFile(r.RuntimeFile, runtimeData)
177	+	}
178	+	}
179	+
180	+	func (r *Runner) DBDoneScan() {
181	+	r.ScanObj.CurrentModule = "done"
182	+	r.ScanObj.RunningTime = r.RunningTime
183	+
184	+	r.ScanObj.DoneStep = r.TotalSteps
185	+	r.ScanObj.IsDone = true
186	+	r.ScanObj.IsRunning = false
187	+
188	+	utils.DebugF("[DB] Done the scan: %v -- %v", r.ScanObj.InputName, r.ScanObj.TaskName)
189	+	database.DBUpdateScan(&r.ScanObj)
190	+	}
191	+
192	+	func (r *Runner) DBUpdateTarget() {
193	+	database.DBUpdateTarget(&r.TargetObj)
194	+	}
195	+
196	+	func (r *Runner) DBNewReports(module libs.Module) {
197	+	if r.Opt.NoDB {
198	+	return
199	+	}
200	+
201	+	r.ScanObj.CurrentModule = r.CurrentModule
202	+	r.ScanObj.RunningTime = r.RunningTime
203	+	database.DBUpdateScan(&r.ScanObj)
204	+
205	+	var reports []string
206	+	reports = append(reports, module.Report.Final...)
207	+	reports = append(reports, module.Report.Noti...)
208	+	reports = append(reports, module.Report.Diff...)
209	+
210	+	utils.DebugF("Updating reports")
211	+	for _, report := range reports {
212	+	reportObj := database.Report{
213	+	ReportName: path.Base(report),
214	+	ModulePath: module.ModulePath,
215	+	Module: module.Name,
216	+	ReportPath: report,
217	+	ReportType: "",
218	+	TargetRefer: r.TargetObj.ID,
219	+	}
220	+
221	+	database.NewReport(&reportObj)
222	+	r.TargetObj.Reports = append(r.TargetObj.Reports, reportObj)
223	+	}
224	+	}
225	+

■ ■ ■ ■ ■ ■

core/external.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/execution"
6	+	"github.com/j3ssie/osmedeus/utils"
7	+	"github.com/robertkrimen/otto"
8	+	"path"
9	+	"time"
10	+	)
11	+
12	+	func (r *Runner) LoadExternalScripts() string {
13	+	var output string
14	+	vm := r.VM
15	+
16	+	// special scripts
17	+	vm.Set(Cleaning, func(call otto.FunctionCall) otto.Value {
18	+	execution.Cleaning(call.Argument(0).String(), r.Opt)
19	+	return otto.Value{}
20	+	})
21	+
22	+	// scripts for cleaning modules
23	+	vm.Set(CleanAmass, func(call otto.FunctionCall) otto.Value {
24	+	src := call.Argument(0).String()
25	+	dest := call.Argument(1).String()
26	+	execution.CleanAmass(src, dest)
27	+	return otto.Value{}
28	+	})
29	+
30	+	vm.Set(CleanRustScan, func(call otto.FunctionCall) otto.Value {
31	+	src := call.Argument(0).String()
32	+	dest := call.Argument(1).String()
33	+	execution.CleanRustScan(src, dest)
34	+	return otto.Value{}
35	+	})
36	+
37	+	vm.Set(CleanGoBuster, func(call otto.FunctionCall) otto.Value {
38	+	src := call.Argument(0).String()
39	+	dest := call.Argument(1).String()
40	+	execution.CleanGoBuster(src, dest)
41	+	return otto.Value{}
42	+	})
43	+	vm.Set(CleanMassdns, func(call otto.FunctionCall) otto.Value {
44	+	src := call.Argument(0).String()
45	+	dest := call.Argument(1).String()
46	+	execution.CleanMassdns(src, dest)
47	+	return otto.Value{}
48	+	})
49	+
50	+	vm.Set(CleanSWebanalyze, func(call otto.FunctionCall) otto.Value {
51	+	src := call.Argument(0).String()
52	+	dest := call.Argument(1).String()
53	+	execution.CleanSWebanalyze(src, dest)
54	+	return otto.Value{}
55	+	})
56	+	vm.Set(CleanJSONDnsx, func(call otto.FunctionCall) otto.Value {
57	+	src := call.Argument(0).String()
58	+	dest := call.Argument(1).String()
59	+	execution.CleanJSONDnsx(src, dest)
60	+	return otto.Value{}
61	+	})
62	+
63	+	vm.Set(CleanJSONHttpx, func(call otto.FunctionCall) otto.Value {
64	+	src := call.Argument(0).String()
65	+	dest := call.Argument(1).String()
66	+	execution.CleanJSONHttpx(src, dest)
67	+	return otto.Value{}
68	+	})
69	+
70	+	// Deprecated
71	+	vm.Set(CleanWebanalyze, func(call otto.FunctionCall) otto.Value {
72	+	src := call.Argument(0).String()
73	+	dest := call.Argument(1).String()
74	+	args := call.ArgumentList
75	+
76	+	techSum := path.Join(path.Dir(dest), fmt.Sprintf("tech-overview-%v.txt", r.Target["Workspace"]))
77	+	if len(args) > 3 {
78	+	techSum = args[2].String()
79	+	}
80	+	execution.CleanWebanalyze(src, dest, techSum)
81	+	return otto.Value{}
82	+	})
83	+
84	+	vm.Set(CleanArjun, func(call otto.FunctionCall) otto.Value {
85	+	// src mean folder contain arjun output
86	+	src := call.Argument(0).String()
87	+	dest := call.Argument(1).String()
88	+	execution.CleanArjun(src, dest)
89	+	return otto.Value{}
90	+	})
91	+
92	+	vm.Set(GenNucleiReport, func(call otto.FunctionCall) otto.Value {
93	+	src := call.Argument(0).String()
94	+	dest := call.Argument(1).String()
95	+	args := call.ArgumentList
96	+
97	+	templateFile := ""
98	+	if len(args) >= 3 {
99	+	templateFile = args[2].String()
100	+	}
101	+	execution.GenNucleiReport(r.Opt, src, dest, templateFile)
102	+	return otto.Value{}
103	+	})
104	+
105	+	return output
106	+	}
107	+
108	+	func (r *Runner) LoadGitScripts() string {
109	+	var output string
110	+	vm := r.VM
111	+	options := r.Opt
112	+
113	+	// Clone("[email protected]", "/tmp/dest")
114	+	vm.Set(Clone, func(call otto.FunctionCall) otto.Value {
115	+	execution.GitClone(call.Argument(0).String(), call.Argument(1).String(), false, options)
116	+	return otto.Value{}
117	+	})
118	+	// like clone but delete the destination folder first
119	+	vm.Set(FClone, func(call otto.FunctionCall) otto.Value {
120	+	execution.GitClone(call.Argument(0).String(), call.Argument(1).String(), true, options)
121	+	return otto.Value{}
122	+	})
123	+
124	+	vm.Set(PushResult, func(call otto.FunctionCall) otto.Value {
125	+	for folder := range options.Storages {
126	+	execution.PullResult(folder, options)
127	+	time.Sleep(3 * time.Second)
128	+	execution.PullResult(folder, options)
129	+	commitMess := fmt.Sprintf("%v\|%v\|%v", options.Module.Name, options.Scan.ROptions["Workspace"], utils.GetCurrentDay())
130	+	execution.PushResult(folder, commitMess, options)
131	+	}
132	+	return otto.Value{}
133	+	})
134	+	// push result but specific folder
135	+	vm.Set(PushFolder, func(call otto.FunctionCall) otto.Value {
136	+	folder := call.Argument(0).String()
137	+	execution.PullResult(folder, options)
138	+	time.Sleep(3 * time.Second)
139	+	execution.PullResult(folder, options)
140	+	commitMess := fmt.Sprintf("%v\|%v\|%v", options.Module.Name, options.Scan.ROptions["Workspace"], utils.GetCurrentDay())
141	+	execution.PushResult(folder, commitMess, options)
142	+	return otto.Value{}
143	+	})
144	+
145	+	// push result but specific folder
146	+	vm.Set(PullFolder, func(call otto.FunctionCall) otto.Value {
147	+	folder := call.Argument(0).String()
148	+	execution.PullResult(folder, options)
149	+	time.Sleep(3 * time.Second)
150	+	execution.PullResult(folder, options)
151	+	return otto.Value{}
152	+	})
153	+
154	+	vm.Set(DiffCompare, func(call otto.FunctionCall) otto.Value {
155	+	src := call.Argument(0).String()
156	+	dest := call.Argument(1).String()
157	+	output := call.Argument(2).String()
158	+	execution.DiffCompare(src, dest, output, options)
159	+	return otto.Value{}
160	+	})
161	+
162	+	vm.Set(GitDiff, func(call otto.FunctionCall) otto.Value {
163	+	args := call.ArgumentList
164	+	src := args[0].String()
165	+	output := call.Argument(1).String()
166	+	history := "1"
167	+	if len(args) < 2 {
168	+	history = call.Argument(2).String()
169	+	}
170	+	execution.GitDiff(src, output, history, options)
171	+	return otto.Value{}
172	+	})
173	+	vm.Set(LoopGitDiff, func(call otto.FunctionCall) otto.Value {
174	+	args := call.ArgumentList
175	+	src := args[0].String()
176	+	output := call.Argument(1).String()
177	+	execution.LoopGitDiff(src, output, options)
178	+	return otto.Value{}
179	+	})
180	+
181	+	vm.Set(GetFileFromCDN, func(call otto.FunctionCall) otto.Value {
182	+	args := call.ArgumentList
183	+	src := args[0].String()
184	+	output := args[1].String()
185	+	execution.GetFileFromCDN(options, src, output)
186	+	return otto.Value{}
187	+	})
188	+
189	+	vm.Set(GetWSFromCDN, func(call otto.FunctionCall) otto.Value {
190	+	args := call.ArgumentList
191	+	src := args[0].String()
192	+	output := args[1].String()
193	+	execution.GetWSFromCDN(options, src, output)
194	+	return otto.Value{}
195	+	})
196	+
197	+	vm.Set(DownloadFile, func(call otto.FunctionCall) otto.Value {
198	+	args := call.ArgumentList
199	+	src := args[0].String()
200	+	output := args[1].String()
201	+	execution.DownloadFile(options, src, output)
202	+	return otto.Value{}
203	+	})
204	+	/* --- Gitlab API --- */
205	+
206	+	// CreateRepo("repo-name")
207	+	// CreateRepo("repo-name", "tags")
208	+	vm.Set(CreateRepo, func(call otto.FunctionCall) otto.Value {
209	+	args := call.ArgumentList
210	+	repoName := args[0].String()
211	+	tags := ""
212	+	if len(args) > 1 {
213	+	tags = args[1].String()
214	+	}
215	+	execution.CreateGitlabRepo(repoName, tags, options)
216	+	return otto.Value{}
217	+	})
218	+
219	+	vm.Set(DeleteRepo, func(call otto.FunctionCall) otto.Value {
220	+	args := call.ArgumentList
221	+	repoName := args[0].String()
222	+	execution.DeleteRepo(repoName, 0, options)
223	+	return otto.Value{}
224	+	})
225	+	vm.Set(DeleteRepoByPid, func(call otto.FunctionCall) otto.Value {
226	+	args := call.ArgumentList
227	+	pid, err := args[0].ToInteger()
228	+	if err != nil {
229	+	return otto.Value{}
230	+	}
231	+	execution.DeleteRepo("", int(pid), options)
232	+	return otto.Value{}
233	+	})
234	+	vm.Set(ListProjects, func(call otto.FunctionCall) otto.Value {
235	+	args := call.ArgumentList
236	+	if len(args) > 0 {
237	+	uid, err := args[0].ToInteger()
238	+	if err == nil {
239	+	execution.ListProjects(int(uid), options)
240	+	}
241	+	return otto.Value{}
242	+	}
243	+	execution.ListProjects(0, options)
244	+	return otto.Value{}
245	+	})
246	+
247	+	return output
248	+	}
249	+

■ ■ ■ ■ ■ ■

core/flow.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/j3ssie/osmedeus/utils"
6	+	"github.com/thoas/go-funk"
7	+	"path"
8	+	"path/filepath"
9	+	"strings"
10	+
11	+	"github.com/j3ssie/osmedeus/libs"
12	+	)
13	+
14	+	// ListFlow list all available mode
15	+	func ListFlow(options libs.Options) (result []string) {
16	+	modePath := path.Join(options.Env.WorkFlowsFolder, "/*.yaml")
17	+	result, err := filepath.Glob(modePath)
18	+	if err != nil {
19	+	return result
20	+	}
21	+	return result
22	+	}
23	+
24	+	// SelectFlow select flow to run
25	+	func SelectFlow(flowName string, options libs.Options) []string {
26	+	flows := ListFlow(options)
27	+	var selectedFlow []string
28	+
29	+	// absolute path like -f customflows/general.yaml
30	+	if strings.HasSuffix(flowName, ".yaml") {
31	+	if utils.FileExists(flowName) {
32	+	selectedFlow = append(selectedFlow, flowName)
33	+	return selectedFlow
34	+	}
35	+	}
36	+
37	+	// -f test
38	+	if !strings.Contains(flowName, ",") {
39	+	selectedFlow = append(selectedFlow, singleMode(flowName, flows)...)
40	+	}
41	+
42	+	// -f test1,test2
43	+	flowNames := strings.Split(flowName, ",")
44	+	for _, item := range flowNames {
45	+	selectedFlow = append(selectedFlow, singleMode(item, flows)...)
46	+	}
47	+
48	+	// default custom flow folder
49	+	if !utils.FileExists(flowName) {
50	+	flowName = path.Join(options.Env.WorkFlowsFolder, "default-flows", flowName)
51	+	if utils.FileExists(flowName) {
52	+	selectedFlow = append(selectedFlow, flowName)
53	+	} else if utils.FileExists(flowName + ".yaml") {
54	+	flowName = flowName + ".yaml"
55	+	selectedFlow = append(selectedFlow, flowName)
56	+	}
57	+	}
58	+
59	+	selectedFlow = funk.UniqString(selectedFlow)
60	+	return selectedFlow
61	+	}
62	+
63	+	func singleMode(modeName string, modes []string) (selectedMode []string) {
64	+	for _, mode := range modes {
65	+	basemodeName := strings.TrimRight(strings.TrimRight(filepath.Base(mode), "yaml"), ".")
66	+	// select workflow file in workflow directory
67	+	if strings.ToLower(basemodeName) == strings.ToLower(modeName) {
68	+	selectedMode = append(selectedMode, mode)
69	+	}
70	+	}
71	+	return selectedMode
72	+	}
73	+
74	+	// ListModules list all available module
75	+	func ListModules(options libs.Options) (modules []string) {
76	+	modePath := path.Join(options.Env.WorkFlowsFolder, "general/*.yaml")
77	+	if options.Flow.Type != "" {
78	+	modePath = path.Join(options.Env.WorkFlowsFolder, fmt.Sprintf("%v/*.yaml", options.Flow.Type))
79	+	}
80	+	if strings.HasSuffix(options.Scan.Flow, ".yaml") {
81	+	if options.Flow.Type == "" {
82	+	options.Flow.Type = "general"
83	+	}
84	+	modePath = path.Join(path.Dir(options.Scan.Flow), options.Flow.Type) + "/*.yaml"
85	+	}
86	+	modules, err := filepath.Glob(modePath)
87	+	if err != nil {
88	+	return modules
89	+	}
90	+	return modules
91	+	}
92	+
93	+	// SelectModules return list of modules name
94	+	func SelectModules(moduleNames []string, options libs.Options) []string {
95	+	if strings.Contains(options.Flow.Type, "{{.") {
96	+	options.Flow.Type = ResolveData(options.Flow.Type, options.Scan.ROptions)
97	+	}
98	+	modules := ListModules(options)
99	+	var selectedModules []string
100	+	for _, item := range moduleNames {
101	+	selectedModules = append(selectedModules, singleSelectModule(item, modules)...)
102	+	}
103	+	selectedModules = funk.UniqString(selectedModules)
104	+
105	+	utils.DebugF("Select module name %v: %v", moduleNames, selectedModules)
106	+	return selectedModules
107	+	}
108	+
109	+	func singleSelectModule(moduleName string, modules []string) (selectedModules []string) {
110	+	for _, module := range modules {
111	+	baseModuleName := strings.Trim(strings.TrimRight(filepath.Base(module), "yaml"), ".")
112	+	if strings.ToLower(baseModuleName) == strings.ToLower(moduleName) {
113	+	selectedModules = append(selectedModules, module)
114	+	}
115	+	}
116	+	return selectedModules
117	+	}
118	+
119	+	// DirectSelectModule select module from ~/.osmedeus/core/workflow/plugins/
120	+	func DirectSelectModule(options libs.Options, moduleName string) string {
121	+	// got absolutely path
122	+	if utils.FileExists(moduleName) {
123	+	return moduleName
124	+	}
125	+
126	+	// select in cloud folder first if we're running the cloud scan
127	+	// ~/.osmedeus/core/workflow/cloud-modules/
128	+	basePlugin := path.Join(options.Env.WorkFlowsFolder, "cloud-modules")
129	+	modulePath := path.Join(basePlugin, moduleName)
130	+	if utils.FileExists(modulePath) {
131	+	utils.DebugF("Load module path: %v", modulePath)
132	+	return modulePath
133	+	}
134	+
135	+	modulePath = path.Join(basePlugin, moduleName+".yaml")
136	+	if utils.FileExists(modulePath) {
137	+	utils.DebugF("Load module path: %v", modulePath)
138	+	return modulePath
139	+	}
140	+
141	+	// ~/.osmedeus/core/workflow/default-modules/
142	+	basePlugin = path.Join(options.Env.WorkFlowsFolder, "default-modules")
143	+	modulePath = path.Join(basePlugin, moduleName)
144	+	utils.DebugF("Load module path: %v", modulePath)
145	+	if utils.FileExists(modulePath) {
146	+	return modulePath
147	+	}
148	+
149	+	modulePath = path.Join(basePlugin, moduleName+".yaml")
150	+	utils.DebugF("Load module path: %v", modulePath)
151	+	if utils.FileExists(modulePath) {
152	+	return modulePath
153	+	}
154	+	utils.DebugF("No plugin found with: %v", moduleName)
155	+	return ""
156	+	}
157	+
158	+	// ListScripts list all available mode
159	+	func ListScripts(options libs.Options) (result []string) {
160	+	modePath := path.Join(options.Env.OseFolder, "/*.js")
161	+	result, err := filepath.Glob(modePath)
162	+	if err != nil {
163	+	return result
164	+	}
165	+
166	+	modePath = path.Join(options.Env.OseFolder, "//.js")
167	+	DepthResult, err := filepath.Glob(modePath)
168	+	if err == nil {
169	+	result = append(result, DepthResult...)
170	+	}
171	+	return result
172	+	}
173	+
174	+	func SelectScript(scriptName string, options libs.Options) string {
175	+	scripts := ListScripts(options)
176	+	for _, script := range scripts {
177	+	if strings.Contains(scriptName, "/") {
178	+	if strings.HasSuffix(script, scriptName) \|\| strings.HasSuffix(script, scriptName+".js") {
179	+	return script
180	+	}
181	+	}
182	+
183	+	compareName := path.Base(script)
184	+	if compareName == scriptName {
185	+	return script
186	+	}
187	+	if compareName == fmt.Sprintf("%s.js", scriptName) {
188	+	return script
189	+	}
190	+	}
191	+	return ""
192	+	}
193	+

■ ■ ■ ■ ■ ■

core/import.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/Jeffail/gabs/v2"
6	+	"github.com/j3ssie/osmedeus/database"
7	+	"github.com/j3ssie/osmedeus/utils"
8	+	jsoniter "github.com/json-iterator/go"
9	+	"github.com/robertkrimen/otto"
10	+	"github.com/spf13/cast"
11	+	"path"
12	+	"path/filepath"
13	+	"strings"
14	+	)
15	+
16	+	func (r *Runner) LoadImportScripts() string {
17	+	var output string
18	+
19	+	// DB scripts
20	+
21	+	r.VM.Set("ImportSubdomain", func(call otto.FunctionCall) otto.Value {
22	+	src := call.Argument(0).String()
23	+	r.ImportSubdomain(src)
24	+	return otto.Value{}
25	+	})
26	+
27	+	r.VM.Set("ImportDns", func(call otto.FunctionCall) otto.Value {
28	+	src := call.Argument(0).String()
29	+	r.ImportDns(src)
30	+	return otto.Value{}
31	+	})
32	+
33	+	r.VM.Set("ImportTech", func(call otto.FunctionCall) otto.Value {
34	+	src := call.Argument(0).String()
35	+	r.ImportTech(src)
36	+	return otto.Value{}
37	+	})
38	+
39	+	r.VM.Set("ImportHTTPJson", func(call otto.FunctionCall) otto.Value {
40	+	src := call.Argument(0).String()
41	+	r.ImportHTTPJson(src)
42	+	return otto.Value{}
43	+	})
44	+
45	+	r.VM.Set("ImportScreenShotJson", func(call otto.FunctionCall) otto.Value {
46	+	src := call.Argument(0).String()
47	+	r.ImportScreenShotJson(src)
48	+	return otto.Value{}
49	+	})
50	+
51	+	r.VM.Set("ImportPortJson", func(call otto.FunctionCall) otto.Value {
52	+	src := call.Argument(0).String()
53	+	r.ImportPortJson(src)
54	+	return otto.Value{}
55	+	})
56	+
57	+	r.VM.Set("ImportJaelesVulnJson", func(call otto.FunctionCall) otto.Value {
58	+	src := call.Argument(0).String()
59	+	r.ImportJaelesVulnJson(src)
60	+	return otto.Value{}
61	+	})
62	+
63	+	r.VM.Set("ImportNucleiVulnJson", func(call otto.FunctionCall) otto.Value {
64	+	src := call.Argument(0).String()
65	+	r.ImportNucleiVulnJson(src)
66	+	return otto.Value{}
67	+	})
68	+
69	+	r.VM.Set("ImportDirectoryJson", func(call otto.FunctionCall) otto.Value {
70	+	src := call.Argument(0).String()
71	+	r.ImportDirectoryJson(src)
72	+	return otto.Value{}
73	+	})
74	+
75	+	r.VM.Set("ImportLinks", func(call otto.FunctionCall) otto.Value {
76	+	src := call.Argument(0).String()
77	+	r.ImportLinks(src)
78	+	return otto.Value{}
79	+	})
80	+
81	+	r.VM.Set("ImportArchive", func(call otto.FunctionCall) otto.Value {
82	+	src := call.Argument(0).String()
83	+	r.ImportArchive(src)
84	+	return otto.Value{}
85	+	})
86	+
87	+	r.VM.Set("ImportIPRange", func(call otto.FunctionCall) otto.Value {
88	+	src := call.Argument(0).String()
89	+	r.ImportIPRange(src)
90	+	return otto.Value{}
91	+	})
92	+
93	+	r.VM.Set("ImportCred", func(call otto.FunctionCall) otto.Value {
94	+	src := call.Argument(0).String()
95	+	r.ImportCred(src)
96	+	return otto.Value{}
97	+	})
98	+
99	+	r.VM.Set("ImportCert", func(call otto.FunctionCall) otto.Value {
100	+	src := call.Argument(0).String()
101	+	r.ImportCert(src)
102	+	return otto.Value{}
103	+	})
104	+
105	+	r.VM.Set("ImportCloudBrute", func(call otto.FunctionCall) otto.Value {
106	+	src := call.Argument(0).String()
107	+	r.ImportCloudBrute(src)
108	+	return otto.Value{}
109	+	})
110	+
111	+	r.VM.Set("SummaryTarget", func(call otto.FunctionCall) otto.Value {
112	+	r.ScanObj.SummaryTarget()
113	+	return otto.Value{}
114	+	})
115	+
116	+	return output
117	+	}
118	+
119	+	func (r *Runner) ImportSubdomain(src string) {
120	+	if !utils.FileExists(src) {
121	+	utils.ErrorF("file not found: %v", src)
122	+	return
123	+	}
124	+
125	+	domains := utils.ReadingLines(src)
126	+	var objs []database.Asset
127	+
128	+	for _, domain := range domains {
129	+	domain = strings.TrimSpace(domain)
130	+	if domain == "" {
131	+	continue
132	+	}
133	+
134	+	obj := database.Asset{
135	+	AssetValue: domain,
136	+	//Dns: nil,
137	+	HTTP: nil,
138	+	Directory: nil,
139	+	Vulnerability: nil,
140	+	ScanRefer: r.ScanObj.ID,
141	+	TargetRefer: r.ScanObj.TargetRefer,
142	+	}
143	+	objs = append(objs, obj)
144	+	}
145	+	database.ImportAssets(objs)
146	+	}
147	+
148	+	func (r *Runner) ImportDns(src string) {
149	+	if !utils.FileExists(src) {
150	+	utils.ErrorF("file not found: %v", src)
151	+	return
152	+	}
153	+
154	+	content := utils.ReadingLines(src)
155	+	for _, line := range content {
156	+	line = strings.TrimSpace(line)
157	+	if line == "" \|\| !strings.Contains(line, " ") {
158	+	continue
159	+	}
160	+
161	+	raw := strings.Split(line, " ")
162	+	domain := strings.Trim(raw[0], ".")
163	+	dnsType := strings.TrimSpace(raw[1])
164	+	dnsValue := strings.Trim(raw[2], ".")
165	+
166	+	if strings.TrimSpace(domain) == "" \|\| strings.TrimSpace(dnsValue) == "" {
167	+	continue
168	+	}
169	+
170	+	dnsChecksum := utils.GenHash(fmt.Sprintf("%s-%s-%s", domain, dnsType, dnsValue))
171	+
172	+	obj := database.Dns{
173	+	Domain: domain,
174	+	DnsType: dnsType,
175	+	DnsValue: dnsValue,
176	+	DnsChecksum: dnsChecksum,
177	+	ScanRefer: r.ScanObj.ID,
178	+	TargetRefer: r.ScanObj.TargetRefer,
179	+	}
180	+	obj.Create()
181	+	}
182	+	}
183	+
184	+	func (r *Runner) ImportTech(src string) {
185	+	if !utils.FileExists(src) {
186	+	utils.ErrorF("file not found: %v", src)
187	+	return
188	+	}
189	+
190	+	content := utils.ReadingLines(src)
191	+	for _, line := range content {
192	+	if !strings.Contains(line, ";;") {
193	+	utils.ErrorF("Invalid format: %v", line)
194	+	continue
195	+	}
196	+
197	+	// data should be domain\|%v;;techs\|%v
198	+	domain := strings.TrimPrefix(strings.Split(line, ";;")[0], "domain\|")
199	+	techs := strings.TrimPrefix(strings.Split(line, ";;")[1], "techs\|")
200	+	if strings.TrimSpace(techs) == "" {
201	+	utils.ErrorF("Invalid format: %v", line)
202	+	continue
203	+	}
204	+
205	+	obj := database.Asset{
206	+	AssetValue: domain,
207	+	Technology: techs,
208	+	IsAlive: true,
209	+	ScanRefer: r.ScanObj.ID,
210	+	TargetRefer: r.ScanObj.TargetRefer,
211	+	}
212	+
213	+	obj.UpdateTech()
214	+	}
215	+	}
216	+
217	+	func (r *Runner) ImportHTTPJson(src string) {
218	+	if !utils.FileExists(src) {
219	+	utils.ErrorF("file not found: %v", src)
220	+	return
221	+	}
222	+
223	+	content := utils.ReadingLines(src)
224	+
225	+	baseResult, _ := filepath.Abs(src)
226	+	baseResult = path.Dir(baseResult)
227	+	utils.DebugF("Set Base Dir for content: %v", baseResult)
228	+
229	+	for _, line := range content {
230	+	if strings.TrimSpace(line) == "" {
231	+	continue
232	+	}
233	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
234	+	if err != nil {
235	+	continue
236	+	}
237	+
238	+	// {"url":"https://sso-na.tesla.com","title":"Blank Title","checksum":"527ef0b39a78caf74f54ca5b2ffb59bcfe688685","content_file":"overview/contents/https___sso-na.tesla.com.txt","status":"302","time":"0.046268348","length":"90","redirect":"https://teamchatgl.tesla.com/"}
239	+	URL := jsonParsed.S("url").Data().(string)
240	+	title := jsonParsed.S("title").Data().(string)
241	+	checksum := jsonParsed.S("checksum").Data().(string)
242	+	contentPath := jsonParsed.S("content_file").Data().(string)
243	+	status := jsonParsed.S("status").Data().(string)
244	+	length := jsonParsed.S("length").Data().(string)
245	+	redirect := jsonParsed.S("redirect").Data().(string)
246	+
247	+	if !utils.FileExists(contentPath) {
248	+	contentPath = path.Join(baseResult, contentPath)
249	+	}
250	+
251	+	// base64 content
252	+	data := "No-Content"
253	+	if !strings.Contains(contentPath, "No-Content") {
254	+	//utils.DebugF("Reading HTML content: %v", contentPath)
255	+	data = utils.ImageAsBase64(contentPath)
256	+	}
257	+
258	+	obj := database.HTTP{
259	+	URL: URL,
260	+	Title: title,
261	+	Checksum: checksum,
262	+	StatusCode: cast.ToInt(status),
263	+	ContentLength: cast.ToInt(length),
264	+	HTTPContent: data,
265	+	Redirect: redirect,
266	+	ScanRefer: r.ScanObj.ID,
267	+	TargetRefer: r.ScanObj.TargetRefer,
268	+	}
269	+	obj.CreateHTTP()
270	+	}
271	+	}
272	+
273	+	func (r *Runner) ImportScreenShotJson(src string) {
274	+	if !utils.FileExists(src) {
275	+	utils.ErrorF("file not found: %v", src)
276	+	return
277	+	}
278	+
279	+	baseResult, _ := filepath.Abs(src)
280	+	baseResult = path.Dir(baseResult)
281	+	content := utils.ReadingLines(src)
282	+	utils.DebugF("Set Base Dir for screenshot: %v", baseResult)
283	+
284	+	for _, line := range content {
285	+	if strings.TrimSpace(line) == "" {
286	+	continue
287	+	}
288	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
289	+	if err != nil {
290	+	continue
291	+	}
292	+
293	+	URL := jsonParsed.S("url").Data().(string)
294	+	imgPath := jsonParsed.S("image").Data().(string)
295	+	tech := jsonParsed.S("tech").Data().(string)
296	+	if tech != "" {
297	+	//domain, _ := utils.GetDomain(URL)
298	+	utils.DebugF("more tech: %v", tech)
299	+	//database.NewTech(wsObj, domain, tech)
300	+	}
301	+
302	+	if !utils.FileExists(imgPath) {
303	+	imgPath = path.Join(baseResult, imgPath)
304	+	}
305	+	imgData := utils.ImageAsBase64(imgPath)
306	+
307	+	//database.NewScreenshot(wsObj, URL, data)
308	+
309	+	obj := database.HTTP{
310	+	URL: URL,
311	+	//Title: tittle,
312	+	//Checksum: "",
313	+	//StatusCode: 0,
314	+	//ContentLength: 0,
315	+	//HTTPContent: "",
316	+	ScreenShotData: imgData,
317	+	ScanRefer: r.ScanObj.ID,
318	+	TargetRefer: r.ScanObj.TargetRefer,
319	+	}
320	+	obj.CreateScreenShot()
321	+	}
322	+	}
323	+
324	+	type PortObj struct {
325	+	Protocol string
326	+	PortID string
327	+	State string
328	+	Service struct {
329	+	Name string
330	+	Product string
331	+	Cpe string
332	+	}
333	+	Script struct {
334	+	ID string
335	+	Output string
336	+	}
337	+	}
338	+
339	+	func (r *Runner) ImportPortJson(src string) {
340	+	if !utils.FileExists(src) {
341	+	utils.ErrorF("file not found: %v", src)
342	+	return
343	+	}
344	+	content := utils.ReadingLines(src)
345	+
346	+	for _, line := range content {
347	+	if strings.TrimSpace(line) == "" {
348	+	continue
349	+	}
350	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
351	+	if err != nil {
352	+	continue
353	+	}
354	+
355	+	var portObj []PortObj
356	+
357	+	ipAddress := jsonParsed.S("IPAddress").Data().(string)
358	+	rawPorts := jsonParsed.S("Ports").Bytes()
359	+	err = jsoniter.Unmarshal(rawPorts, &portObj)
360	+	if err != nil \|\| len(portObj) == 0 {
361	+	continue
362	+	}
363	+
364	+	var ports []string
365	+	for _, p := range portObj {
366	+	info := fmt.Sprintf("%v/%v/%v", p.PortID, p.Protocol, p.Service.Product)
367	+	ports = append(ports, strings.Trim(info, "/"))
368	+	}
369	+	if len(ports) == 0 {
370	+	continue
371	+	}
372	+
373	+	obj := database.Dns{
374	+	DnsValue: ipAddress,
375	+	DnsType: "A",
376	+	Ports: strings.Join(ports, ","),
377	+	ScanRefer: r.ScanObj.ID,
378	+	TargetRefer: r.ScanObj.TargetRefer,
379	+	}
380	+	obj.UpdatePort()
381	+	}
382	+	}
383	+
384	+	// ImportJaelesVulnJson import new asset to DB
385	+	func (r *Runner) ImportJaelesVulnJson(src string) {
386	+	if !utils.FileExists(src) {
387	+	utils.ErrorF("file not found: %v", src)
388	+	return
389	+	}
390	+	content := utils.ReadingLines(src)
391	+	baseDir := path.Dir(src)
392	+	for _, line := range content {
393	+	if strings.TrimSpace(line) == "" {
394	+	continue
395	+	}
396	+
397	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
398	+	if err != nil {
399	+	utils.ErrorF("Error parse JSON Data")
400	+	continue
401	+	}
402	+	raw := jsonParsed.S("OutputFile").Data().(string)
403	+	reportPath := raw
404	+	if !utils.FileExists(reportPath) {
405	+	reportPath = path.Join(baseDir, raw)
406	+	if !utils.FileExists(reportPath) {
407	+	reportPath = path.Join(path.Dir(baseDir), raw)
408	+	}
409	+	}
410	+	vulnContent := utils.GetFileContent(reportPath)
411	+
412	+	// parse Data as JSON
413	+	jsonParsed, err = gabs.ParseJSON([]byte(vulnContent))
414	+	if err != nil {
415	+	utils.ErrorF("Error parse JSON Data")
416	+	continue
417	+	}
418	+
419	+	obj := database.Vulnerability{
420	+	URL: jsonParsed.S("URL").Data().(string),
421	+	VulnRequest: jsonParsed.S("Req").Data().(string),
422	+	VulnResponse: jsonParsed.S("Res").Data().(string),
423	+	DetectionString: jsonParsed.S("DetectionString").Data().(string),
424	+	VulnerabilityTitle: jsonParsed.S("SignName").Data().(string),
425	+	SignatureID: jsonParsed.S("SignID").Data().(string),
426	+	Confidence: jsonParsed.S("Confidence").Data().(string),
427	+	Severity: jsonParsed.S("Risk").Data().(string),
428	+	Source: "Jaeles",
429	+	//VulnChecksum: utils.GenHash(vulnData),
430	+	//PluginScan: SelectScanID(workspace, pluginName, scanID),
431	+	//Target: SelectScanByWS(workspace),
432	+	//Asset: SelectAssetByData(domain),
433	+	ScanRefer: r.ScanObj.ID,
434	+	TargetRefer: r.ScanObj.TargetRefer,
435	+	}
436	+	obj.Create()
437	+	}
438	+	}
439	+
440	+	// ImportNucleiVulnJson import new asset to DB
441	+	func (r *Runner) ImportNucleiVulnJson(src string) {
442	+	if !utils.FileExists(src) {
443	+	utils.ErrorF("file not found: %v", src)
444	+	return
445	+	}
446	+	content := utils.ReadingLines(src)
447	+	for _, line := range content {
448	+	if strings.TrimSpace(line) == "" {
449	+	continue
450	+	}
451	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
452	+	if err != nil {
453	+	utils.ErrorF("Error parse JSON Data")
454	+	continue
455	+	}
456	+
457	+	obj := database.Vulnerability{
458	+	URL: jsonParsed.S("host").Data().(string),
459	+	VulnRequest: utils.Base64Encode(jsonParsed.S("request").Data().(string)),
460	+	VulnResponse: utils.Base64Encode(jsonParsed.S("response").Data().(string)),
461	+	DetectionString: jsonParsed.S("matched").Data().(string),
462	+
463	+	SignatureID: jsonParsed.S("templateID").Data().(string),
464	+	Confidence: "Tentative",
465	+	VulnerabilityTitle: jsonParsed.S("info", "name").Data().(string),
466	+	Severity: jsonParsed.S("info", "severity").Data().(string),
467	+	Source: "Nuclei",
468	+	TargetRefer: r.ScanObj.TargetRefer,
469	+
470	+	ScanRefer: r.ScanObj.ID,
471	+	}
472	+	obj.Create()
473	+	}
474	+	}
475	+
476	+	// ImportDirectoryJson import new asset to DB
477	+	func (r *Runner) ImportDirectoryJson(src string) {
478	+	if !utils.FileExists(src) {
479	+	utils.ErrorF("file not found: %v", src)
480	+	return
481	+	}
482	+	content := utils.ReadingLines(src)
483	+	for _, line := range content {
484	+	if strings.TrimSpace(line) == "" \|\| !strings.Contains(line, "url") {
485	+	continue
486	+	}
487	+
488	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
489	+	if err != nil {
490	+	utils.ErrorF("Error parse JSON Data")
491	+	continue
492	+	}
493	+
494	+	// some parser here
495	+	URL := jsonParsed.S("url").Data().(string)
496	+	redirect, ok := jsonParsed.S("redirectlocation").Data().(string)
497	+	if ok {
498	+	redirect = ""
499	+	}
500	+
501	+	obj := database.Directory{
502	+	URL: URL,
503	+	Status: cast.ToInt(jsonParsed.S("status").Data()),
504	+	ContentLength: cast.ToInt(jsonParsed.S("length").Data()),
505	+	Words: cast.ToInt(jsonParsed.S("words").Data()),
506	+	RedirectURL: redirect,
507	+	ScanRefer: r.ScanObj.ID,
508	+	TargetRefer: r.ScanObj.TargetRefer,
509	+	}
510	+	obj.Create()
511	+	}
512	+	}
513	+
514	+	//
515	+	//// UpdateJSONDns update dns in db
516	+	//func UpdateJSONDns(src string, options libs.Options) {
517	+	// content := utils.ReadingLines(src)
518	+	// if len(content) == 0 {
519	+	// utils.ErrorF("File not found: %v", src)
520	+	// return
521	+	// }
522	+	//
523	+	// utils.DebugF("Update JSON DNS: %v", src)
524	+	// //wsObj := database.SelectScanByWS(options.Scan.ROptions["Workspace"])
525	+	// target := options.Scan.ROptions["Workspace"]
526	+	//
527	+	// for _, line := range content {
528	+	// jsonParsed, err := gabs.ParseJSON([]byte(line))
529	+	// if err != nil {
530	+	// continue
531	+	// }
532	+	//
533	+	// domain, ok := jsonParsed.S("host").Data().(string)
534	+	// if !ok {
535	+	// continue
536	+	// }
537	+	//
538	+	// // filtered some unrelated domain
539	+	// if !strings.HasSuffix(domain, fmt.Sprintf(".%s", target)) {
540	+	// if domain != target {
541	+	// continue
542	+	// }
543	+	// }
544	+	//
545	+	// data := "N/A"
546	+	// isError := jsonParsed.S("status_code").Data().(string)
547	+	// if isError == "NXDOMAIN" {
548	+	// //database.NewAssetWithDns(wsObj, domain, data)
549	+	// continue
550	+	// }
551	+	// var results []string
552	+	// a := jsonParsed.S("a")
553	+	// if a != nil {
554	+	// for _, record := range a.Children() {
555	+	// data := fmt.Sprintf("A/%s", cast.ToString(record.Data()))
556	+	// results = append(results, data)
557	+	// }
558	+	// }
559	+	//
560	+	// cname := jsonParsed.S("cname")
561	+	// if cname != nil {
562	+	// for _, record := range cname.Children() {
563	+	// data := fmt.Sprintf("CNAME/%s", cast.ToString(record.Data()))
564	+	// results = append(results, data)
565	+	// }
566	+	// }
567	+	//
568	+	// mx := jsonParsed.S("mx")
569	+	// if mx != nil {
570	+	// for _, record := range mx.Children() {
571	+	// data := fmt.Sprintf("MX/%s", cast.ToString(record.Data()))
572	+	// results = append(results, data)
573	+	// }
574	+	// }
575	+	//
576	+	// ns := jsonParsed.S("ns")
577	+	// if ns != nil {
578	+	// for _, record := range ns.Children() {
579	+	// data := fmt.Sprintf("NS/%s", cast.ToString(record.Data()))
580	+	// results = append(results, data)
581	+	// }
582	+	// }
583	+	// results = funk.UniqString(results)
584	+	// data = strings.Join(results, ";;")
585	+	// //database.NewAssetWithDns(wsObj, domain, data)
586	+	// }
587	+	//}
588	+

■ ■ ■ ■ ■ ■

core/import_mics.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/Jeffail/gabs/v2"
6	+	"github.com/j3ssie/osmedeus/database"
7	+	"github.com/j3ssie/osmedeus/utils"
8	+	jsoniter "github.com/json-iterator/go"
9	+	"github.com/spf13/cast"
10	+	"gorm.io/datatypes"
11	+	"net/url"
12	+	"strings"
13	+	)
14	+
15	+	func (r *Runner) StartScanNoti() error {
16	+	data, err := jsoniter.Marshal(&r.ScanObj)
17	+	if err != nil {
18	+	return fmt.Errorf("err marshal object: %v", err)
19	+	}
20	+	noti := database.Notification{
21	+	NotificationType: "start",
22	+	NotificationSource: "scan",
23	+	NewData: datatypes.JSON(data),
24	+	ObjRefer: r.ScanObj.ID,
25	+	ScanRefer: r.ScanObj.ID,
26	+	TargetRefer: r.TargetObj.ID,
27	+	}
28	+	return noti.Create()
29	+	}
30	+
31	+	func (r *Runner) ScanDoneNoti() error {
32	+	data, err := jsoniter.Marshal(&r.ScanObj)
33	+	if err != nil {
34	+	return fmt.Errorf("err marshal object: %v", err)
35	+	}
36	+	noti := database.Notification{
37	+	NotificationType: "done",
38	+	NotificationSource: "scan",
39	+	NewData: datatypes.JSON(data),
40	+	ObjRefer: r.ScanObj.ID,
41	+	ScanRefer: r.ScanObj.ID,
42	+	TargetRefer: r.TargetObj.ID,
43	+	}
44	+	return noti.Create()
45	+	}
46	+
47	+	// ImportLinks import new link to DB
48	+	func (r *Runner) ImportLinks(src string) {
49	+	if !utils.FileExists(src) {
50	+	utils.ErrorF("file not found: %v", src)
51	+	return
52	+	}
53	+
54	+	content := utils.ReadingLines(src)
55	+	for _, line := range content {
56	+	if strings.TrimSpace(line) == "" {
57	+	continue
58	+	}
59	+	// {"input":"http://academy.live-test.shopee.io","source":"body","type":"url","output":"http://academy.live-test.shopee.io","status":307,"length":7}
60	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
61	+	if err != nil {
62	+	utils.ErrorF("Error parse JSON Data")
63	+	continue
64	+	}
65	+
66	+	obj := database.Link{
67	+	LinkValue: jsonParsed.S("output").Data().(string),
68	+	LinkSource: jsonParsed.S("source").Data().(string),
69	+	URL: jsonParsed.S("input").Data().(string),
70	+	LinkType: jsonParsed.S("type").Data().(string),
71	+	ScanRefer: r.ScanObj.ID,
72	+	TargetRefer: r.TargetObj.ID,
73	+	}
74	+	obj.Create()
75	+	}
76	+	}
77	+
78	+	func (r *Runner) ImportArchive(src string) {
79	+	if !utils.FileExists(src) {
80	+	utils.ErrorF("file not found: %v", src)
81	+	return
82	+	}
83	+
84	+	content := utils.ReadingLines(src)
85	+	for _, line := range content {
86	+	if strings.TrimSpace(line) == "" {
87	+	continue
88	+	}
89	+
90	+	// ignore root domain
91	+	u, err := url.Parse(line)
92	+	if err != nil \|\| u.Path == "" \|\| u.Path == "/" {
93	+	continue
94	+	}
95	+
96	+	obj := database.Archive{
97	+	ArchiveValue: line,
98	+	ArchiveChecksum: utils.GenHash(line),
99	+	ScanRefer: r.ScanObj.ID,
100	+	TargetRefer: r.TargetObj.ID,
101	+	}
102	+	obj.Create()
103	+	}
104	+	}
105	+
106	+	func (r *Runner) ImportIPRange(src string) {
107	+	if !utils.FileExists(src) {
108	+	utils.ErrorF("file not found: %v", src)
109	+	return
110	+	}
111	+
112	+	content := utils.ReadingLines(src)
113	+	for _, line := range content {
114	+	if strings.TrimSpace(line) == "" {
115	+	continue
116	+	}
117	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
118	+	if err != nil {
119	+	utils.ErrorF("Error parse JSON Data")
120	+	continue
121	+	}
122	+
123	+	utils.DebugF("Processing: %v", line)
124	+	obj := database.IPRange{
125	+	ASNumber: cast.ToString(jsonParsed.S("Number").Data()),
126	+	Country: cast.ToString(jsonParsed.S("CountryCode").Data()),
127	+	Value: cast.ToString(jsonParsed.S("CIDR").Data()),
128	+	Info: cast.ToString(jsonParsed.S("Description").Data()),
129	+	Amount: cast.ToUint(jsonParsed.S("Count").Data()),
130	+	ScanRefer: r.ScanObj.ID,
131	+	TargetRefer: r.TargetObj.ID,
132	+	}
133	+	obj.Create()
134	+	}
135	+	}
136	+
137	+	func (r *Runner) ImportCert(src string) {
138	+	if !utils.FileExists(src) {
139	+	utils.ErrorF("file not found: %v", src)
140	+	return
141	+	}
142	+
143	+	content := utils.ReadingLines(src)
144	+	for _, line := range content {
145	+	if strings.TrimSpace(line) == "" {
146	+	continue
147	+	}
148	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
149	+	if err != nil {
150	+	utils.ErrorF("Error parse JSON Data")
151	+	continue
152	+	}
153	+
154	+	domain := cast.ToString(jsonParsed.S("Domain").Data())
155	+	isWildCard := false
156	+	if strings.Contains(domain, "*.") {
157	+	isWildCard = true
158	+	domain = strings.TrimLeft(domain, "*.")
159	+	}
160	+
161	+	obj := database.CertInfo{
162	+	Domain: domain,
163	+	CertInfo: cast.ToString(jsonParsed.S("CertInfo").Data()),
164	+	OrgInfo: cast.ToString(jsonParsed.S("OrgInfo").Data()),
165	+	IsWildcard: isWildCard,
166	+	TargetRefer: r.TargetObj.ID,
167	+	}
168	+	obj.Create()
169	+	}
170	+	}
171	+
172	+	func (r *Runner) ImportCred(src string) {
173	+	if !utils.FileExists(src) {
174	+	utils.ErrorF("file not found: %v", src)
175	+	return
176	+	}
177	+
178	+	content := utils.ReadingLines(src)
179	+	for _, line := range content {
180	+	if strings.TrimSpace(line) == "" {
181	+	continue
182	+	}
183	+
184	+	jsonParsed, err := gabs.ParseJSON([]byte(line))
185	+	if err != nil {
186	+	utils.ErrorF("Error parse JSON Data")
187	+	continue
188	+	}
189	+
190	+	email := cast.ToString(jsonParsed.S("email").Data())
191	+	credID := cast.ToString(jsonParsed.S("id").Data())
192	+	username := cast.ToString(jsonParsed.S("username").Data())
193	+	password := cast.ToString(jsonParsed.S("password").Data())
194	+	hashedPassword := cast.ToString(jsonParsed.S("hashed_password").Data())
195	+	name := cast.ToString(jsonParsed.S("name").Data())
196	+	ipAddress := cast.ToString(jsonParsed.S("ip_address").Data())
197	+	phone := cast.ToString(jsonParsed.S("phone").Data())
198	+	source := cast.ToString(jsonParsed.S("database_name").Data())
199	+
200	+	obj := database.Credential{
201	+	CredID: credID,
202	+	Email: email,
203	+	Username: username,
204	+	Password: password,
205	+	HashedPassword: hashedPassword,
206	+	Name: name,
207	+	Phone: phone,
208	+	IPAddress: ipAddress,
209	+	Source: source,
210	+	TargetRefer: r.TargetObj.ID,
211	+	ScanRefer: r.ScanObj.ID,
212	+	}
213	+	obj.Create()
214	+	}
215	+	}
216	+
217	+	func (r *Runner) ImportCloudBrute(src string) {
218	+	if !utils.FileExists(src) {
219	+	utils.ErrorF("file not found: %v", src)
220	+	return
221	+	}
222	+
223	+	content := utils.ReadingLines(src)
224	+	for _, line := range content {
225	+	if strings.TrimSpace(line) == "" {
226	+	continue
227	+	}
228	+
229	+	if !strings.Contains(line, " - ") {
230	+	continue
231	+	}
232	+
233	+	cloudDomain := strings.Split(line, " - ")[1]
234	+	status := strings.Split(line, " - ")[0]
235	+	if strings.Contains(status, ": ") {
236	+	status = strings.Split(strings.Split(line, " - ")[0], ": ")[1]
237	+	}
238	+
239	+	obj := database.CloudBrute{
240	+	Status: status,
241	+	CloudDomain: cloudDomain,
242	+	RawData: line,
243	+	ScanRefer: r.ScanObj.ID,
244	+	TargetRefer: r.TargetObj.ID,
245	+	}
246	+	obj.Create()
247	+	}
248	+	}
249	+

■ ■ ■ ■ ■ ■

core/mode_test.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"testing"
6	+
7	+	"github.com/j3ssie/osmedeus/libs"
8	+	)
9	+
10	+	func TestListMode(t *testing.T) {
11	+	var options libs.Options
12	+	options.Env.WorkFlowsFolder = "~/go/src/github.com/j3ssie/osmedeus/workflow/"
13	+	result := ListFlow(options)
14	+	fmt.Println(result)
15	+	if len(result) == 0 {
16	+	t.Errorf("Error ListMode")
17	+	}
18	+
19	+	selectedMode := SelectFlow("general", options)
20	+	fmt.Println(selectedMode)
21	+	if len(selectedMode) == 0 {
22	+	t.Errorf("Error selectedMode")
23	+	}
24	+	}
25	+

■ ■ ■ ■ ■ ■

core/parse.go

1	+	package core
2	+
3	+	import (
4	+	"bytes"
5	+	"fmt"
6	+	"github.com/Jeffail/gabs/v2"
7	+	"github.com/spf13/cast"
8	+	"golang.org/x/net/publicsuffix"
9	+	"io/ioutil"
10	+	"net/url"
11	+	"os"
12	+	"path"
13	+	"path/filepath"
14	+	"strings"
15	+	"text/template"
16	+	"time"
17	+
18	+	"github.com/Shopify/yaml"
19	+	"github.com/j3ssie/osmedeus/libs"
20	+	"github.com/j3ssie/osmedeus/utils"
21	+	)
22	+
23	+	// ResolveData resolve template from signature file
24	+	func ResolveData(format string, data map[string]string) string {
25	+	t := template.Must(template.New("").Parse(format))
26	+	buf := &bytes.Buffer{}
27	+	err := t.Execute(buf, data)
28	+	if err != nil {
29	+	return format
30	+	}
31	+	return buf.String()
32	+	}
33	+
34	+	// ParseFlow parse mode file
35	+	func ParseFlow(flowFile string) (libs.Flow, error) {
36	+	utils.DebugF("Parsing workflow at: %v", flowFile)
37	+	var flow libs.Flow
38	+	yamlFile, err := ioutil.ReadFile(flowFile)
39	+	if err != nil {
40	+	utils.ErrorF("YAML parsing err: %v -- #%v ", flowFile, err)
41	+	return flow, err
42	+	}
43	+	err = yaml.Unmarshal(yamlFile, &flow)
44	+	if err != nil {
45	+	utils.ErrorF("Error unmarshal: %v -- %v", flowFile, err)
46	+	return flow, err
47	+	}
48	+
49	+	if flow.Usage != "" && strings.Contains(flow.Usage, "{{.this_file}}") {
50	+	flow.Usage = strings.ReplaceAll(flow.Usage, "{{.this_file}}", flowFile)
51	+	}
52	+	return flow, nil
53	+	}
54	+
55	+	// ParseModules parse module file
56	+	func ParseModules(moduleFile string) (libs.Module, error) {
57	+	utils.DebugF("Parsing module at: %v", moduleFile)
58	+
59	+	var module libs.Module
60	+
61	+	yamlFile, err := ioutil.ReadFile(moduleFile)
62	+	if err != nil {
63	+	utils.ErrorF("YAML parsing err: %v -- #%v ", moduleFile, err)
64	+	return module, err
65	+	}
66	+	err = yaml.Unmarshal(yamlFile, &module)
67	+	if err != nil {
68	+	utils.ErrorF("Error unmarshal: %v -- %v", moduleFile, err)
69	+	return module, err
70	+	}
71	+	module.ModulePath = moduleFile
72	+	if module.Usage != "" && strings.Contains(module.Usage, "{{.this_file}}") {
73	+	module.Usage = strings.ReplaceAll(module.Usage, "{{.this_file}}", moduleFile)
74	+	}
75	+	return module, err
76	+	}
77	+
78	+	// ParseInputFormat format input
79	+	func ParseInputFormat(raw string, options libs.Options) map[string]string {
80	+	target := make(map[string]string)
81	+	target["RawFormat"] = raw
82	+
83	+	jsonParsed, err := gabs.ParseJSON([]byte(raw))
84	+	if err != nil {
85	+	return target
86	+	}
87	+
88	+	// parse Target first if found one
89	+	rawURL, ok := jsonParsed.ChildrenMap()["Target"]
90	+	if !ok {
91	+	panic("missing Target in special input")
92	+	}
93	+
94	+	target = ParseInput(cast.ToString(rawURL.Data()), options)
95	+
96	+	// override the whole thing
97	+	for k, v := range jsonParsed.ChildrenMap() {
98	+	target[k] = fmt.Sprintf("%v", v.Data())
99	+	}
100	+
101	+	return target
102	+	}
103	+
104	+	// ParseInput parse input for routine
105	+	func ParseInput(raw string, options libs.Options) map[string]string {
106	+	ROptions := ParseTarget(raw)
107	+	if options.EnableFormatInput {
108	+	// avoid loop hole
109	+	options.EnableFormatInput = false
110	+	ROptions = ParseInputFormat(raw, options)
111	+	return ROptions
112	+	}
113	+	// some data stuff
114	+	dir, err := os.Getwd()
115	+	if err == nil {
116	+	ROptions["CWD"] = dir
117	+	}
118	+	ROptions["WSCDN"] = options.Cdn.WSURL
119	+	ROptions["CDN"] = options.Cdn.URL
120	+	ROptions["Date"] = time.Now().Format("2006-01-02")
121	+	ROptions["TS"] = utils.GetCurrentDay()
122	+
123	+	/* --- start to load default Env --- */
124	+	// ~/osmedeus-base
125	+	ROptions["BaseFolder"] = utils.NormalizePath(strings.TrimLeft(options.Env.BaseFolder, "/"))
126	+	ROptions["Plugins"] = options.Env.BinariesFolder
127	+	ROptions["Binaries"] = options.Env.BinariesFolder
128	+
129	+	ROptions["Data"] = options.Env.DataFolder
130	+	ROptions["Workflow"] = options.Env.WorkFlowsFolder
131	+	ROptions["Scripts"] = options.Env.WorkFlowsFolder
132	+	ROptions["Cloud"] = options.Env.CloudConfigFolder
133	+
134	+	// ~/.osmedeus/clouds
135	+	ROptions["CWorkspaces"] = options.Env.CloudDataFolder
136	+	ROptions["Workspaces"] = options.Env.WorkspacesFolder
137	+	if options.Scan.BaseWorkspace != "" {
138	+	ROptions["Workspaces"] = options.Scan.BaseWorkspace
139	+	}
140	+	ROptions["Storages"] = options.Env.StoragesFolder
141	+	/* --- end of load default Env --- */
142	+
143	+	ROptions["Workspace"] = utils.CleanPath(raw)
144	+	if options.Scan.CustomWorkspace != "" {
145	+	ROptions["Workspace"] = utils.CleanPath(options.Scan.CustomWorkspace)
146	+	}
147	+	ROptions["Output"] = path.Join(ROptions["Workspaces"], ROptions["Workspace"])
148	+
149	+	// params in workflow file
150	+	if len(options.Flow.Params) > 0 {
151	+	for _, param := range options.Flow.Params {
152	+	for k, v := range param {
153	+	v = ResolveData(v, ROptions)
154	+	if strings.HasPrefix(v, "~/") {
155	+	v = utils.NormalizePath(v)
156	+	}
157	+	ROptions[k] = v
158	+	}
159	+	}
160	+	}
161	+
162	+	return ROptions
163	+	}
164	+
165	+	// MoreParams get more params from module or cli
166	+	func MoreParams(module libs.Module, options *libs.Options) {
167	+	ROptions := options.Scan.ROptions
168	+	if len(ROptions) == 0 {
169	+	ROptions = make(map[string]string)
170	+	}
171	+	// params from module
172	+	if len(module.Params) > 0 {
173	+	for _, param := range module.Params {
174	+	for k, v := range param {
175	+	// skip params if override: false
176	+	_, exist := ROptions[k]
177	+	if module.Permanent && exist {
178	+	utils.DebugF("Skip Override param: %v", k)
179	+	continue
180	+	}
181	+
182	+	v = ResolveData(v, ROptions)
183	+	if strings.HasPrefix(v, "~/") {
184	+	v = utils.NormalizePath(v)
185	+	}
186	+	ROptions[k] = v
187	+	}
188	+	}
189	+	}
190	+
191	+	// more params
192	+	if len(options.Scan.Params) > 0 {
193	+	params := ParseParams(options.Scan.Params)
194	+	if len(params) > 0 {
195	+	for k, v := range params {
196	+	v = ResolveData(v, ROptions)
197	+	ROptions[k] = v
198	+	}
199	+	}
200	+	}
201	+
202	+	options.Scan.ROptions = ROptions
203	+	}
204	+
205	+	// ParseParams parse more params from cli
206	+	func ParseParams(rawParams []string) map[string]string {
207	+	params := make(map[string]string)
208	+	for _, item := range rawParams {
209	+	if strings.Contains(item, "=") {
210	+	data := strings.Split(item, "=")
211	+	params[data[0]] = strings.Replace(item, data[0]+"=", "", -1)
212	+	}
213	+	}
214	+	return params
215	+	}
216	+
217	+	// ParseTarget parsing target and some variable for template
218	+	func ParseTarget(raw string) map[string]string {
219	+	target := make(map[string]string)
220	+	if raw == "" {
221	+	return target
222	+	}
223	+	target["Target"] = raw
224	+	u, err := url.Parse(raw)
225	+
226	+	// something wrong so parsing it again
227	+	if err != nil \|\| u.Scheme == "" \|\| strings.Contains(u.Scheme, ".") {
228	+	raw = fmt.Sprintf("https://%v", raw)
229	+	u, err = url.Parse(raw)
230	+	if err != nil {
231	+	return target
232	+	}
233	+	// fmt.Println("parse again")
234	+	}
235	+	var hostname string
236	+	var query string
237	+	port := u.Port()
238	+	// var domain string
239	+	domain := u.Hostname()
240	+
241	+	query = u.RawQuery
242	+	if u.Port() == "" {
243	+	if strings.Contains(u.Scheme, "https") {
244	+	port = "443"
245	+	} else {
246	+	port = "80"
247	+	}
248	+
249	+	hostname = u.Hostname()
250	+	} else {
251	+	// ignore common port in Host
252	+	if u.Port() == "443" \|\| u.Port() == "80" {
253	+	hostname = u.Hostname()
254	+	} else {
255	+	hostname = u.Hostname() + ":" + u.Port()
256	+	}
257	+	}
258	+
259	+	target["Scheme"] = u.Scheme
260	+	target["Path"] = u.Path
261	+	target["Domain"] = domain
262	+
263	+	target["Org"] = domain
264	+	suffix, ok := publicsuffix.PublicSuffix(domain)
265	+	if ok {
266	+	target["Org"] = strings.Replace(domain, fmt.Sprintf(".%s", suffix), "", -1)
267	+	} else {
268	+	if strings.Contains(domain, ".") {
269	+	parts := strings.Split(domain, ".")
270	+	if len(parts) == 2 {
271	+	target["Org"] = parts[0]
272	+	} else {
273	+	target["Org"] = parts[len(parts)-2]
274	+	}
275	+	}
276	+	}
277	+
278	+	target["Host"] = hostname
279	+	target["Port"] = port
280	+	target["RawQuery"] = query
281	+
282	+	if (target["RawQuery"] != "") && (port == "80" \|\| port == "443") {
283	+	target["URL"] = fmt.Sprintf("%v://%v%v?%v", target["Scheme"], target["Host"], target["Path"], target["RawQuery"])
284	+	} else if port != "80" && port != "443" {
285	+	target["URL"] = fmt.Sprintf("%v://%v:%v%v?%v", target["Scheme"], target["Domain"], target["Port"], target["Path"], target["RawQuery"])
286	+	} else {
287	+	target["URL"] = fmt.Sprintf("%v://%v%v", target["Scheme"], target["Host"], target["Path"])
288	+	}
289	+
290	+	uu, _ := url.Parse(raw)
291	+	target["BaseURL"] = fmt.Sprintf("%v://%v", uu.Scheme, uu.Host)
292	+	target["Extension"] = filepath.Ext(target["BaseURL"])
293	+
294	+	return target
295	+	}
296	+

■ ■ ■ ■ ■ ■

core/parse_test.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"testing"
6	+	)
7	+
8	+	func TestParseTarget(t *testing.T) {
9	+	fmt.Println("----> INPUT:", "http://example.com")
10	+	result := ParseTarget("http://exmaple.com")
11	+	fmt.Println(result)
12	+	if len(result) == 0 {
13	+	t.Errorf("Error RunMasscan")
14	+	}
15	+	// case 2
16	+	fmt.Println("----> INPUT:", "example.com")
17	+	result = ParseTarget("exmaple.com")
18	+	fmt.Println(result)
19	+	if len(result) == 0 {
20	+	t.Errorf("Error RunMasscan")
21	+	}
22	+
23	+	// case 2
24	+	fmt.Println("----> INPUT:", "http://exmaple.com/123?q=1")
25	+	result = ParseTarget("http://exmaple.com/123?q=1")
26	+	fmt.Println(result)
27	+	if len(result) == 0 {
28	+	t.Errorf("Error RunMasscan")
29	+	}
30	+
31	+	// case 2
32	+	fmt.Println("----> INPUT:", "1.2.3.4")
33	+	result = ParseTarget("1.2.3.4")
34	+	fmt.Println(result)
35	+	if len(result) == 0 {
36	+	t.Errorf("Error RunMasscan")
37	+	}
38	+
39	+	// case 2
40	+	fmt.Println("----> INPUT:", "1.2.3.4/24")
41	+	result = ParseTarget("1.2.3.4/24")
42	+	fmt.Println(result)
43	+	if len(result) == 0 {
44	+	t.Errorf("Error RunMasscan")
45	+	}
46	+	}
47	+

■ ■ ■ ■ ■ ■

core/reference.go

1	+	package core
2	+
3	+	/* File to store all the script for better reference */
4	+
5	+	const (
6	+	SetVar = "SetVar"
7	+	GetOSEnv = "GetOSEnv"
8	+	)
9	+
10	+	const (
11	+	RRSync = "RRSync"
12	+	Clone = "Clone"
13	+	FClone = "FClone"
14	+	PushResult = "PushResult"
15	+	PushFolder = "PushFolder"
16	+	PullFolder = "PullFolder"
17	+	DiffCompare = "DiffCompare"
18	+	GitDiff = "GitDiff"
19	+	LoopGitDiff = "LoopGitDiff"
20	+	GetFileFromCDN = "GetFileFromCDN"
21	+	GetWSFromCDN = "GetWSFromCDN"
22	+	DownloadFile = "DownloadFile"
23	+	// for gitlab API only
24	+	CreateRepo = "CreateRepo"
25	+	DeleteRepo = "DeleteRepo"
26	+	DeleteRepoByPid = "DeleteRepoByPid"
27	+	ListProjects = "ListProjects"
28	+	)
29	+
30	+	const (
31	+	Cleaning = "Cleaning"
32	+	CleanAmass = "CleanAmass"
33	+	CleanRustScan = "CleanRustScan"
34	+	CleanGoBuster = "CleanGoBuster"
35	+	CleanMassdns = "CleanMassdns"
36	+	CleanSWebanalyze = "CleanSWebanalyze"
37	+	CleanJSONDnsx = "CleanJSONDnsx"
38	+	CleanWebanalyze = "CleanWebanalyze"
39	+	CleanArjun = "CleanArjun"
40	+	GenNucleiReport = "GenNucleiReport"
41	+	CleanJSONHttpx = "CleanJSONHttpx"
42	+	)
43	+
44	+	const (
45	+	// noti for slack
46	+	StartNoti = "StartNoti"
47	+	DoneNoti = "DoneNoti"
48	+	ReportNoti = "ReportNoti"
49	+	DiffNoti = "DiffNoti"
50	+	CustomNoti = "CustomNoti"
51	+	NotiFile = "NotiFile"
52	+	WebHookNoti = "WebHookNoti"
53	+	// noti for telegram
54	+	TeleMess = "TeleMess"
55	+	TeleMessWrap = "TeleMessWrap"
56	+	TeleMessByFile = "TeleMessByFile"
57	+	TeleSendFile = "TeleSendFile"
58	+	)
59	+
60	+	const (
61	+	ExecCmd = "ExecCmd"
62	+	ExecCmdB = "ExecCmdB"
63	+	ExecCmdWithOutput = "ExecCmdWithOutput"
64	+	ExecContain = "ExecContain"
65	+	)
66	+
67	+	const (
68	+	Sleep = "Sleep"
69	+	Exit = "Exit"
70	+	CastToInt = "CastToInt"
71	+	StripSlash = "StripSlash"
72	+	Printf = "Printf"
73	+	)
74	+
75	+	const (
76	+	SortU = "SortU"
77	+	SplitFile = "SplitFile"
78	+	Append = "Append"
79	+	Copy = "Copy"
80	+	CreateFolder = "CreateFolder"
81	+	DeleteFile = "DeleteFile"
82	+	DeleteFolder = "DeleteFolder"
83	+	SplitFileByPart = "SplitFileByPart"
84	+	FileLength = "FileLength"
85	+	IsFile = "IsFile"
86	+	EmptyDir = "EmptyDir"
87	+	EmptyFile = "EmptyFile"
88	+	ReadLines = "ReadLines"
89	+	)
90	+

■ ■ ■ ■ ■ ■

core/routine.go

1	+	package core
2	+
3	+	import (
4	+	"context"
5	+	"errors"
6	+	"fmt"
7	+	"path"
8	+	"strings"
9	+	"sync"
10	+	"time"
11	+
12	+	"github.com/spf13/cast"
13	+
14	+	"github.com/fatih/color"
15	+	"github.com/j3ssie/osmedeus/execution"
16	+	"github.com/j3ssie/osmedeus/libs"
17	+	"github.com/j3ssie/osmedeus/utils"
18	+	"github.com/jinzhu/copier"
19	+	"github.com/panjf2000/ants"
20	+	)
21	+
22	+	// RunModule run the module
23	+	func (r *Runner) RunModule(module libs.Module, options libs.Options) {
24	+	// get more params
25	+	MoreParams(module, &options)
26	+	// get reports path
27	+	options.Module = ResolveReports(module, options)
28	+	r.LoadEngineScripts()
29	+
30	+	// check if resume enable or not
31	+	if (options.Resume \|\| module.Resume) && !module.Forced {
32	+	if CheckResume(options) {
33	+	utils.BlockF(module.Name, "Resume detected")
34	+	return
35	+	}
36	+	}
37	+
38	+	r.CurrentModule = module.Name
39	+	timeStart := time.Now()
40	+	utils.BannerF("MODULES", fmt.Sprintf("%v - %v", module.Name, options.Module.Desc))
41	+
42	+	// create report record first because I don't want to wait for them to show up in UI until the module done
43	+	r.DBNewReports(module)
44	+
45	+	// pre-run
46	+	utils.BlockF(module.Name, "Running prepare scripts")
47	+	r.RunScripts(module.PreRun, options)
48	+
49	+	// main part
50	+	utils.BlockF(module.Name, "Start run main steps")
51	+	err := r.RunSteps(module.Steps, options)
52	+	if err != nil {
53	+	utils.BadBlockF(module.Name, fmt.Sprintf("got exit call"))
54	+	}
55	+
56	+	// post-run
57	+	utils.BlockF(module.Name, "Running conclusion scripts")
58	+	r.RunScripts(module.PostRun, options)
59	+
60	+	// print the reports file
61	+	utils.PrintLine()
62	+	printReports(options)
63	+
64	+	// create report record first because we don't want to wait it show up in UI until the module done
65	+	r.DBNewReports(module)
66	+
67	+	// estimate time
68	+	elapsedTime := time.Since(timeStart).Seconds()
69	+	utils.BlockF("Elapsed Time", fmt.Sprintf("Done module %v in %v", color.HiCyanString(module.Name), color.HiMagentaString("%vs", elapsedTime)))
70	+	r.RunningTime += cast.ToInt(elapsedTime)
71	+	utils.PrintLine()
72	+	r.DBUpdateScan()
73	+	r.DBUpdateTarget()
74	+	}
75	+
76	+	// RunScripts run list of scripts
77	+	func (r *Runner) RunScripts(scripts []string, options libs.Options) string {
78	+	if options.Timeout != "" {
79	+	timeout := utils.CalcTimeout(options.Timeout)
80	+	utils.DebugF("Run scripts with %v seconds timeout", timeout)
81	+	r.RunScriptsWithTimeOut(options.Timeout, scripts, options)
82	+	return ""
83	+	}
84	+
85	+	for _, script := range scripts {
86	+	outScript := r.RunScript(script, options)
87	+	if strings.Contains(outScript, "exit") {
88	+	return outScript
89	+	}
90	+	}
91	+	return ""
92	+	}
93	+
94	+	// RunScriptsWithTimeOut run list of scripts with timeout
95	+	func (r *Runner) RunScriptsWithTimeOut(timeoutRaw string, scripts []string, options libs.Options) string {
96	+	timeout := utils.CalcTimeout(timeoutRaw)
97	+	utils.DebugF("Run scripts with %v seconds timeout", timeout)
98	+
99	+	c := context.Background()
100	+	deadline := time.Now().Add(time.Duration(timeout) * time.Second)
101	+	c, cancel := context.WithDeadline(c, deadline)
102	+	defer cancel()
103	+
104	+	go func() {
105	+	for _, script := range scripts {
106	+	outScript := r.RunScript(script, options)
107	+	if strings.Contains(outScript, "exit") {
108	+	return
109	+	}
110	+	}
111	+	cancel()
112	+	}()
113	+
114	+	select {
115	+	case <-c.Done():
116	+	utils.DebugF("Scripts done")
117	+	return ""
118	+	case <-time.After(time.Duration(timeout) * time.Second):
119	+	utils.BadBlockF("timeout", fmt.Sprintf("Scripts got timeout after %v", color.HiMagentaString(timeoutRaw)))
120	+	}
121	+	return ""
122	+	}
123	+
124	+	// RunScript really run a script
125	+	func (r *Runner) RunScript(script string, options libs.Options) string {
126	+	execScript := ResolveData(script, options.Scan.ROptions)
127	+	return r.ExecScript(execScript)
128	+	}
129	+
130	+	// RunSteps run list of steps
131	+	func (r *Runner) RunSteps(steps []libs.Step, options libs.Options) error {
132	+	var stepOut string
133	+	for _, step := range steps {
134	+	r.DoneStep += 1
135	+	if step.Timeout != "" {
136	+	step.Timeout = ResolveData(step.Timeout, options.Scan.ROptions)
137	+	// timeout should be: 30, 30m, 1h
138	+	timeout := utils.CalcTimeout(step.Timeout)
139	+	if timeout != 0 {
140	+	stepOut, _ = r.RunStepWithTimeout(timeout, step, options)
141	+	if strings.Contains(stepOut, "exit") {
142	+	return errors.New("got exit call")
143	+	}
144	+	continue
145	+	}
146	+	}
147	+
148	+	stepOut, _ = r.RunStep(step, options)
149	+	if strings.Contains(stepOut, "exit") {
150	+	return errors.New("got exit call")
151	+	}
152	+	}
153	+	return nil
154	+	}
155	+
156	+	// RunStepWithTimeout run step with timeout
157	+	func (r *Runner) RunStepWithTimeout(timeout int, step libs.Step, options libs.Options) (string, error) {
158	+	utils.DebugF("Run step with %v seconds timeout", timeout)
159	+	prefix := fmt.Sprintf("timeout -k 1m %vs ", timeout)
160	+
161	+	// prepare the os command with prefix timeout first
162	+	var preFixCommands []string
163	+	for _, command := range step.Commands {
164	+	preFixCommand := command
165	+	if !strings.Contains(command, "timeout") {
166	+	preFixCommand = prefix + command
167	+	}
168	+	preFixCommands = append(preFixCommands, preFixCommand)
169	+	}
170	+	step.Commands = preFixCommands
171	+
172	+	// override global timeout
173	+	options.Timeout = step.Timeout
174	+	output, _ := r.RunStep(step, options)
175	+	return output, nil
176	+	}
177	+
178	+	// RunStep really run a step
179	+	func (r *Runner) RunStep(step libs.Step, options libs.Options) (string, error) {
180	+	var output string
181	+	if step.Label != "" {
182	+	utils.BlockF("Start-Step", color.HiCyanString(step.Label))
183	+	}
184	+
185	+	// checking required file
186	+	err := r.CheckRequired(step.Required, r.Opt)
187	+	if err != nil {
188	+	return output, errors.New("missing requirements")
189	+	}
190	+
191	+	// check conditions and run reverse step
192	+	err = r.CheckCondition(step.Conditions, r.Opt)
193	+	if err != nil {
194	+	if len(step.RCommands) == 0 && len(step.RScripts) == 0 {
195	+	return output, errors.New("conditions not met")
196	+	}
197	+
198	+	// run reverse commands
199	+	utils.InforF("Condition false, run the reverse commands")
200	+	if len(step.RCommands) > 0 {
201	+	RunCommands(step.RCommands, step.Std, r.Opt)
202	+	}
203	+	// run reverse scripts
204	+	if len(step.RScripts) > 0 {
205	+	output = r.RunScripts(step.RScripts, r.Opt)
206	+	if strings.Contains(output, "exit") {
207	+	return output, nil
208	+	}
209	+	}
210	+	return output, nil
211	+	}
212	+
213	+	if step.Source == "" {
214	+	if len(step.Commands) > 0 {
215	+	RunCommands(step.Commands, step.Std, r.Opt)
216	+	}
217	+	if len(step.Scripts) > 0 {
218	+	output = r.RunScripts(step.Scripts, r.Opt)
219	+	if strings.Contains(output, "exit") {
220	+	return output, nil
221	+	}
222	+	}
223	+
224	+	// run ose here
225	+	if len(step.Ose) > 0 {
226	+	for _, ose := range step.Ose {
227	+	r.RunOse(ose)
228	+	}
229	+	}
230	+
231	+	// post scripts
232	+	if len(step.PConditions) > 0 \|\| len(step.PScripts) > 0 {
233	+	err := r.CheckCondition(step.PConditions, r.Opt)
234	+	if err == nil {
235	+	if len(step.PScripts) > 0 {
236	+	r.RunScripts(step.PScripts, r.Opt)
237	+	}
238	+	}
239	+	}
240	+	if step.Label != "" {
241	+	utils.BlockF("Done-Step", color.HiCyanString(step.Label))
242	+	}
243	+	return output, nil
244	+	}
245	+
246	+	////// Start to run step but in loop mode
247	+
248	+	source := ResolveData(step.Source, r.Target)
249	+	utils.DebugF("Run step with Source: %v", source)
250	+	data := utils.ReadingLines(source)
251	+	if len(data) <= 0 {
252	+	return output, errors.New("missing source")
253	+	}
254	+	if step.Threads != "" {
255	+	step.Threads = ResolveData(step.Threads, r.Target)
256	+	step.Parallel = cast.ToInt(step.Threads)
257	+	}
258	+	if step.Parallel == 0 {
259	+	step.Parallel = 1
260	+	}
261	+
262	+	// skip concurrency part
263	+	if step.Parallel == 1 {
264	+	for index, line := range data {
265	+	r.Target["line"] = line
266	+	r.Target["line_id"] = fmt.Sprintf("%v-%v", path.Base(line), index)
267	+	r.Target["_id_"] = fmt.Sprintf("%v", index)
268	+	r.Target["_line_"] = execution.StripName(line)
269	+	if len(step.Commands) > 0 {
270	+	RunCommands(step.Commands, step.Std, options)
271	+	}
272	+
273	+	if len(step.Ose) > 0 {
274	+	for _, ose := range step.Ose {
275	+	r.RunOse(ose)
276	+	}
277	+	}
278	+
279	+	if len(step.Scripts) > 0 {
280	+	r.RunScripts(step.Scripts, options)
281	+	}
282	+
283	+	// post scripts
284	+	if len(step.PConditions) > 0 \|\| len(step.PScripts) > 0 {
285	+	err := r.CheckCondition(step.PConditions, options)
286	+	if err == nil {
287	+	if len(step.PScripts) > 0 {
288	+	r.RunScripts(step.PScripts, options)
289	+	}
290	+	}
291	+	}
292	+
293	+	}
294	+
295	+	if step.Label != "" {
296	+	utils.BlockF("Done-Step", color.HiCyanString(step.Label))
297	+	}
298	+	return output, nil
299	+	}
300	+
301	+	/////////////
302	+	// run multiple steps in concurrency mode
303	+
304	+	utils.DebugF("Run step in Parallel: %v", step.Parallel)
305	+	var wg sync.WaitGroup
306	+	p, _ := ants.NewPoolWithFunc(step.Parallel, func(i interface{}) {
307	+	r.startStepJob(i)
308	+	wg.Done()
309	+	}, ants.WithPreAlloc(true))
310	+	defer p.Release()
311	+
312	+	var mu sync.Mutex
313	+	for index, line := range data {
314	+	mu.Lock()
315	+	localOptions := libs.Options{}
316	+	copier.Copy(&localOptions, &options)
317	+	localOptions.Scan.ROptions["line"] = line
318	+	localOptions.Scan.ROptions["line_id"] = fmt.Sprintf("%v-%v", path.Base(line), index)
319	+	localOptions.Scan.ROptions["_id_"] = fmt.Sprintf("%v", index)
320	+	localOptions.Scan.ROptions["_line_"] = execution.StripName(line)
321	+
322	+	// make completely new Step
323	+	localStep := libs.Step{}
324	+
325	+	for _, cmd := range step.Commands {
326	+	localStep.Commands = append(localStep.Commands, ResolveData(cmd, localOptions.Scan.ROptions))
327	+	}
328	+	for _, cmd := range step.RCommands {
329	+	localStep.RCommands = append(localStep.RCommands, ResolveData(cmd, localOptions.Scan.ROptions))
330	+	}
331	+
332	+	if len(step.Ose) > 0 {
333	+	for _, ose := range step.Ose {
334	+	localStep.Ose = append(localStep.Ose, ResolveData(ose, localOptions.Scan.ROptions))
335	+	}
336	+	}
337	+
338	+	for _, script := range step.RScripts {
339	+	localStep.RScripts = append(localStep.RScripts, ResolveData(script, localOptions.Scan.ROptions))
340	+	}
341	+
342	+	for _, script := range step.Scripts {
343	+	localStep.Scripts = append(localStep.Scripts, ResolveData(script, localOptions.Scan.ROptions))
344	+	}
345	+
346	+	for _, script := range step.PConditions {
347	+	localStep.PConditions = append(localStep.PConditions, ResolveData(script, localOptions.Scan.ROptions))
348	+	}
349	+	for _, script := range step.PScripts {
350	+	localStep.PScripts = append(localStep.PScripts, ResolveData(script, localOptions.Scan.ROptions))
351	+	}
352	+
353	+	job := stepJob{
354	+	options: localOptions,
355	+	step: localStep,
356	+	}
357	+	wg.Add(1)
358	+	_ = p.Invoke(job)
359	+	mu.Unlock()
360	+	}
361	+
362	+	wg.Wait()
363	+	if step.Label != "" {
364	+	utils.BlockF("Done-Step", color.HiCyanString(step.Label))
365	+	}
366	+	return output, nil
367	+	}
368	+
369	+	type stepJob struct {
370	+	options libs.Options
371	+	step libs.Step
372	+	}
373	+
374	+	func (r *Runner) startStepJob(j interface{}) {
375	+	job := j.(stepJob)
376	+	localOptions := job.options
377	+	localStep := job.step
378	+
379	+	err := r.CheckCondition(localStep.Conditions, localOptions)
380	+
381	+	if err != nil {
382	+	// run reverse commands
383	+	if len(localStep.RCommands) > 0 {
384	+	RunCommands(localStep.RCommands, localStep.Std, localOptions)
385	+	}
386	+	if len(localStep.RScripts) > 0 {
387	+	r.RunScripts(localStep.RScripts, localOptions)
388	+	}
389	+	} else {
390	+	if len(localStep.Commands) > 0 {
391	+	RunCommands(localStep.Commands, localStep.Std, localOptions)
392	+	}
393	+	}
394	+
395	+	if len(localStep.Ose) > 0 {
396	+	for _, ose := range localStep.Ose {
397	+	r.RunOse(ose)
398	+	}
399	+	}
400	+
401	+	if len(localStep.Scripts) > 0 {
402	+	r.RunScripts(localStep.Scripts, localOptions)
403	+	}
404	+
405	+	// post scripts
406	+	if len(localStep.PConditions) > 0 \|\| len(localStep.PScripts) > 0 {
407	+	err := r.CheckCondition(localStep.PConditions, localOptions)
408	+	if err == nil {
409	+	if len(localStep.PScripts) > 0 {
410	+	r.RunScripts(localStep.PScripts, localOptions)
411	+	}
412	+	}
413	+	}
414	+	}
415	+

■ ■ ■ ■ ■ ■

core/runner.go

1	+	package core
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/fatih/color"
6	+	"github.com/j3ssie/osmedeus/database"
7	+	"github.com/j3ssie/osmedeus/execution"
8	+	"github.com/j3ssie/osmedeus/libs"
9	+	"github.com/j3ssie/osmedeus/utils"
10	+	"github.com/robertkrimen/otto"
11	+	"github.com/thoas/go-funk"
12	+	"os"
13	+	"strings"
14	+	)
15	+
16	+	// Runner runner struct to start a job
17	+	type Runner struct {
18	+	Input string
19	+	Workspace string
20	+
21	+	InputType string // domain, url, ip, cidr or domain-file, url-file, ip-file, cidr-file
22	+	RequiredInput string // this should match with InputType
23	+	IsInvalid bool
24	+
25	+	RoutineType string // module or flow
26	+	RoutineName string // general
27	+	RoutinePath string
28	+	RunnerSource string // cli or api
29	+	RunnerType string // local or cloud
30	+
31	+	Opt libs.Options
32	+
33	+	// use for analytics
34	+	DoneStep int
35	+	TotalSteps int
36	+	RunningTime int
37	+	CurrentModule string
38	+
39	+	DoneFile string
40	+	RuntimeFile string
41	+
42	+	RoutineModules []string
43	+	Routines []libs.Routine
44	+
45	+	// RunTime *otto.Otto
46	+	VM *otto.Otto
47	+
48	+	TargetObj database.Target
49	+	ScanObj database.Scan
50	+
51	+	Target map[string]string
52	+	}
53	+
54	+	// InitRunner init runner
55	+	func InitRunner(input string, opt libs.Options) (Runner, error) {
56	+	var runner Runner
57	+	runner.Input = input
58	+	runner.Opt = opt
59	+	runner.PrepareRoutine()
60	+	runner.InitVM()
61	+
62	+	runner.RunnerSource = "cli"
63	+	// @TODO check if running in cloud
64	+	runner.RunnerType = "local"
65	+	return runner, nil
66	+	}
67	+
68	+	// PrepareWorkflow prepare workflow file
69	+	func (r *Runner) PrepareWorkflow() {
70	+	r.RoutineType = "flow"
71	+	var err error
72	+	flows := SelectFlow(r.Opt.Scan.Flow, r.Opt)
73	+	for _, flow := range flows {
74	+	if strings.TrimSpace(flow) == "" {
75	+	continue
76	+	}
77	+	r.Opt.Flow, err = ParseFlow(flow)
78	+	r.RoutinePath = flow
79	+	if err != nil {
80	+	continue
81	+	}
82	+
83	+	if r.Opt.Flow.NoDB {
84	+	r.Opt.NoDB = true
85	+	}
86	+
87	+	if r.Opt.Flow.Type == "general" {
88	+	r.Opt.Flow.Input = r.Opt.Scan.Input
89	+	}
90	+	// default folder to look for module file if not specify
91	+	r.Opt.Flow.DefaultType = r.Opt.Flow.Type
92	+
93	+	r.Target["FlowPath"] = flow
94	+	r.RequiredInput = r.Opt.Flow.Validator
95	+
96	+	// get more params from flow
97	+	if len(r.Opt.Flow.Params) > 0 {
98	+	for _, params := range r.Opt.Flow.Params {
99	+	for k, v := range params {
100	+	r.Target[k] = v
101	+	}
102	+	}
103	+	}
104	+	}
105	+
106	+	// generate routines
107	+	for _, routine := range r.Opt.Flow.Routines {
108	+	// select module depend on the flow type
109	+	if routine.FlowFolder != "" {
110	+	r.Opt.Flow.Type = routine.FlowFolder
111	+	} else {
112	+	r.Opt.Flow.Type = r.Opt.Flow.DefaultType
113	+	}
114	+
115	+	modules := SelectModules(routine.Modules, r.Opt)
116	+	routine.RoutineName = fmt.Sprintf("flow-%s", r.Opt.Flow.Name)
117	+
118	+	for _, module := range modules {
119	+	parsedModule, err := ParseModules(module)
120	+	if err != nil \|\| parsedModule.Name == "" {
121	+	continue
122	+	}
123	+	r.TotalSteps += len(parsedModule.Steps)
124	+	routine.ParsedModules = append(routine.ParsedModules, parsedModule)
125	+	}
126	+	r.Routines = append(r.Routines, routine)
127	+	}
128	+	}
129	+
130	+	func (r *Runner) PrepareModule() {
131	+	r.RoutineType = "module"
132	+	var err error
133	+	for _, rawModule := range r.Opt.Scan.Modules {
134	+	var routine libs.Routine
135	+
136	+	module := DirectSelectModule(r.Opt, rawModule)
137	+	r.RoutinePath = rawModule
138	+	r.Opt.Module, err = ParseModules(module)
139	+	if err != nil \|\| r.Opt.Module.Name == "" {
140	+	continue
141	+	}
142	+	if r.Opt.Module.NoDB {
143	+	r.Opt.NoDB = true
144	+	}
145	+
146	+	r.Target["FlowPath"] = "direct-module"
147	+	routine.ParsedModules = append(routine.ParsedModules, r.Opt.Module)
148	+	routine.RoutineName = fmt.Sprintf("module-%s", r.Opt.Flow.Name)
149	+	r.Target["Module"] = module
150	+
151	+	r.Routines = append(r.Routines, routine)
152	+
153	+	r.TotalSteps += len(r.Opt.Module.Steps)
154	+	r.RequiredInput = r.Opt.Module.Validator
155	+	}
156	+	}
157	+
158	+	func (r *Runner) PrepareRoutine() {
159	+	// prepare targets
160	+	r.Target = ParseInput(r.Input, r.Opt)
161	+	r.Workspace = r.Target["Workspace"]
162	+
163	+	// take from -m flag
164	+	if len(r.Opt.Scan.Modules) > 0 {
165	+	r.RoutineName = strings.Join(r.Opt.Scan.Modules, "-")
166	+	r.PrepareModule()
167	+	return
168	+	}
169	+
170	+	// take from -f flag
171	+	if r.Opt.Scan.Flow != "" {
172	+	r.RoutineName = r.Opt.Scan.Flow
173	+	r.PrepareWorkflow()
174	+	}
175	+	}
176	+
177	+	func (r *Runner) Start() {
178	+	err := r.Validator()
179	+	if err != nil {
180	+	utils.ErrorF("Input does not match the require type: %v -- %v", r.RequiredInput, r.Input)
181	+	utils.InforF("Use '--nv' if you want to disable input validate")
182	+	return
183	+	}
184	+
185	+	r.Opt.Scan.ROptions = r.Target
186	+	utils.BlockF("Target", r.Input)
187	+	utils.BlockF("Routine", r.RoutineName)
188	+
189	+	// prepare some metadata files
190	+	utils.MakeDir(r.Target["Output"])
191	+	r.DoneFile = r.Target["Output"] + "/done"
192	+	r.RuntimeFile = r.Target["Output"] + "/runtime"
193	+	os.Remove(r.DoneFile)
194	+
195	+	execution.TeleSendMess(r.Opt, fmt.Sprintf("%s -- Start new scan: %s -- %s", r.Opt.Noti.ClientName, r.Opt.Scan.Flow, r.Target["Workspace"]), "#status", false)
196	+
197	+	r.DBNewTarget()
198	+	r.DBNewScan()
199	+	r.StartScanNoti()
200	+
201	+	/////
202	+	/* really start the scan here */
203	+	r.StartRoutine()
204	+	/////
205	+
206	+	BackupWorkspace(r.Opt)
207	+	r.DBDoneScan()
208	+	r.ScanDoneNoti()
209	+
210	+	utils.BlockF("Done", fmt.Sprintf("scan for %v -- %v", r.Input, color.HiMagentaString("%vs", r.RunningTime)))
211	+	utils.WriteToFile(r.DoneFile, "done")
212	+	}
213	+
214	+	// StartRoutine start the scan
215	+	func (r *Runner) StartRoutine() {
216	+	for _, routine := range r.Routines {
217	+	for _, module := range routine.ParsedModules {
218	+	module = ResolveReports(module, r.Opt)
219	+	r.Opt.Module = module
220	+
221	+	// check exclude options
222	+	if funk.ContainsString(r.Opt.Exclude, module.Name) {
223	+	utils.BadBlockF(module.Name, fmt.Sprintf("Module Got Excluded"))
224	+	continue
225	+	}
226	+
227	+	rTimeout := r.Opt.Timeout
228	+	if module.MTimeout != "" {
229	+	rTimeout = ResolveData(module.MTimeout, r.Target)
230	+	}
231	+
232	+	if rTimeout != "" {
233	+	timeout := utils.CalcTimeout(rTimeout)
234	+	if timeout < 43200 {
235	+	// run the module but with timeout
236	+	r.RunModulesWithTimeout(rTimeout, module, r.Opt)
237	+	continue
238	+	}
239	+	}
240	+
241	+	r.RunModule(module, r.Opt)
242	+	}
243	+	}
244	+	}
245	+

1	+	.DS_STORE
2	+	.idea
3	+	out*
4	+	osmedeus
5	+	dist/*
6	+

Update Next Generation