From ef79ed95003d08a0f7aa8cce4b375565d3ada5a2 Mon Sep 17 00:00:00 2001 From: Egbewatt Kokou Mwangah Smith Date: Fri, 22 Mar 2024 11:18:38 -0600 Subject: [PATCH 1/5] codeql & build --- .github/dependabot.yml | 13 +++++++++++++ .github/workflows/build.yml | 34 +++++++++++++++++++++++++++++++++ .github/workflows/codeql.yaml | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/codeql.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c825540 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..0cc676d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,34 @@ +name: Build & Tests +on: + workflow_dispatch: + pull_request: + branches: + - main + +env: + GO_VERSION: 1.21.5 + +jobs: + project_build: + name: TaskWeaver Build + permissions: + contents: read + issues: read + checks: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: go.sum + + - name: Display Go version + run: go version + + - name: Build + run: go build -v ./... diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 0000000..4e7f5fe --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,36 @@ +name: "CodeQL" + +on: + push: + branches: [main, feat/nodes-flow] + pull_request: + branches: [main] + schedule: + - cron: '0 3 * * */2' + +permissions: + security-events: write + actions: read + contents: read + +jobs: + analyse: + name: Analyse + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: go + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:go" \ No newline at end of file From 8a1ecdc68a42091fec1f1a2ae8d8239343036893 Mon Sep 17 00:00:00 2001 From: Egbewatt Kokou Mwangah Smith Date: Fri, 22 Mar 2024 11:19:03 -0600 Subject: [PATCH 2/5] update docs --- README.md | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/README.md b/README.md index aff1f82..ded89ac 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,6 @@ TaskWeaver is a comprehensive platform designed to streamline task management, automate repetitive processes, and enhance overall system efficiency. By providing users with a user-friendly interface and advanced scheduling capabilities, this project empowers Platform Engineering teams to optimize their workflows and reduce manual intervention. -## Table of Contents - -- [Introduction](#introduction) -- [Features](#features) -- [Architecture](#architecture) -- [Plugin System](#plugin-system) -- [Integration with External Systems](#integration-with-external-systems) -- [Research](#research) - -## Introduction - -Task Automation and Scheduling is a critical component of modern Platform Engineering. This project aims to offer a robust solution for defining, scheduling, and executing various tasks, enabling users to automate routine activities, minimize errors, and enhance productivity. From managing simple recurring tasks to orchestrating complex workflows, this platform provides the tools necessary to achieve these goals effectively. - ## Features **Task Definition and Management:** Users can easily define tasks through an intuitive interface or a configuration file. Each task includes execution commands, input parameters, and dependency specifications. The platform allows users to create, update, delete, and monitor their scheduled jobs. @@ -37,20 +24,10 @@ Task Automation and Scheduling is a critical component of modern Platform Engine **Integration with External Systems:** Integration with version control systems (e.g., Git) automates task triggering upon specific branch changes. Additionally, collaboration with cloud providers and container orchestrators maximizes resource utilization. -## Architecture - -The architecture of this platform revolves around distributed job scheduling. It leverages a combination of single-node and cluster setups to efficiently manage tasks. This architecture accommodates various platforms and environments, making it adaptable to Windows, Linux, and macOS operating systems. - -[https://lucid.app/lucidspark/3b091858-07f0-4443-ad49-5b87a11b78e0/edit?invitationId=inv_e966ce87-b4b7-4572-962f-bf8879dee5e3](https://lucid.app/lucidspark/3b091858-07f0-4443-ad49-5b87a11b78e0/edit?invitationId=inv_e966ce87-b4b7-4572-962f-bf8879dee5e3) - ## Plugin System A plugin system enhances the project's extensibility and versatility. It introduces a well-defined plugin interface, a registry for plugin management, dynamic loading capabilities, and user-configurable plugins. This system enables users to define custom task types, integrate with external systems, enhance scheduling options, and expand notification channels. -## Integration with External Systems - -The project integrates seamlessly with external systems, such as Kubernetes (K8s), Apache Kafka, Apache Airflow, and AWS Batch. These integrations provide users with additional options for managing tasks, enhancing scalability, and leveraging industry-standard tools. - ## Research The development of this project involved research and inspiration from various existing systems: From 8f9ab201911bdcb0d602041a29f7d41b3969ba45 Mon Sep 17 00:00:00 2001 From: Egbewatt Kokou Mwangah Smith Date: Tue, 2 Apr 2024 20:31:56 -0600 Subject: [PATCH 3/5] cleanup build --- .github/workflows/build.yml | 6 ++++-- .github/workflows/codeql.yaml | 2 -- ci/scripts/go-build-test.sh | 14 ++++++++++++++ pkg/node/worker.go | 2 +- pkg/weaver/rest/client.go | 2 +- 5 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 ci/scripts/go-build-test.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0cc676d..ec30363 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,5 +30,7 @@ jobs: - name: Display Go version run: go version - - name: Build - run: go build -v ./... + - name: Build & Test + run: | + chmod +x ci/scripts/go-build-test.sh + ci/scripts/go-build-test.sh diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 4e7f5fe..eb40a73 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -1,8 +1,6 @@ name: "CodeQL" on: - push: - branches: [main, feat/nodes-flow] pull_request: branches: [main] schedule: diff --git a/ci/scripts/go-build-test.sh b/ci/scripts/go-build-test.sh new file mode 100644 index 0000000..6d79975 --- /dev/null +++ b/ci/scripts/go-build-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -eu + +USAGE="USAGE: +${0}" + +if [[ $# -ne 0 ]]; then + echo "${USAGE}" >&2 + exit 1 +fi + +go build -v ./... +go test -v ./... \ No newline at end of file diff --git a/pkg/node/worker.go b/pkg/node/worker.go index dcda0e7..f1f1f6c 100644 --- a/pkg/node/worker.go +++ b/pkg/node/worker.go @@ -72,7 +72,7 @@ func (n *WorkerAgent) GenerateCSR() (err error) { ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, } - println(nodeTemplate) + println(nodeTemplate.SignatureAlgorithm) // node certificate signing request template nodeCSRTemplate := x509.CertificateRequest{ diff --git a/pkg/weaver/rest/client.go b/pkg/weaver/rest/client.go index 7dab033..67f39e9 100644 --- a/pkg/weaver/rest/client.go +++ b/pkg/weaver/rest/client.go @@ -36,7 +36,7 @@ func NewClient(config ClientConfig, client *http.Client) (*RESTClient, error) { }, nil } -func (rc *RESTClient) withMiddleware() *RESTClient +func (rc *RESTClient) withMiddleware() *RESTClient { return nil } func (rc *RESTClient) Execute(Req IRequest) IResponse { // Build Request From 650d5d8c920e1027512c8e4e4ba86349f185608a Mon Sep 17 00:00:00 2001 From: Egbewatt Kokou Mwangah Smith Date: Tue, 7 May 2024 18:45:00 -0600 Subject: [PATCH 4/5] get cpu stats --- internal/stats/cpu/cpu.go | 95 ++++++++++++++++++++++++++++++++++ internal/stats/cpu/cpu_test.go | 65 +++++++++++++++++++++++ internal/stats/cpu/data/stat | 20 +++++++ internal/stats/utils/io.go | 15 ++++++ 4 files changed, 195 insertions(+) create mode 100644 internal/stats/cpu/cpu.go create mode 100644 internal/stats/cpu/cpu_test.go create mode 100644 internal/stats/cpu/data/stat create mode 100644 internal/stats/utils/io.go diff --git a/internal/stats/cpu/cpu.go b/internal/stats/cpu/cpu.go new file mode 100644 index 0000000..fcc38f8 --- /dev/null +++ b/internal/stats/cpu/cpu.go @@ -0,0 +1,95 @@ +package cpu + +import ( + "bufio" + "encoding/json" + "log" + "strconv" + "strings" + "taskweaver/internal/stats/utils" +) + +const ( + // stat file path + filePath = "/proc/stat" +) + +// CPU Stats +type Stats struct { + // Time spent in user mode + User float64 `json:"User"` + // Time spent in system mode + System float64 `json:"System"` + Kernel float64 `json:"Kernel"` + // Time spent in the idle task + Idle float64 `json:"Idle"` + // Time waiting for I/O to complete. + IOWait float64 `json:"IOWait"` + // Time spent in user mode with low priority + Nice float64 `json:"Nice"` + IdleTime uint64 `json:"idleTime"` + TotalCPUTime uint64 `json:"totalCPUTime"` + fileOpener utils.FileOpener +} + +// serialized output +func (cpu_stats *Stats) JsonString() string { + // + jsonData, err := json.Marshal(cpu_stats) + if err != nil { + log.Panic("Error:", err) + } + return string(jsonData) +} + +// FileOpener is needed for testing purposes and also coordinate permissions for file access +func NewStats(fileOpener utils.FileOpener) (*Stats, error) { + return &Stats{ + fileOpener: fileOpener, + }, nil +} + +func (s *Stats) Compute() (*Stats, error) { + // compute stats + + statFile, err := s.fileOpener.Open(filePath) + if err != nil { + log.Panicf(`Error opening %s file: %e`, filePath, err) + } + defer statFile.Close() + + // File Scanner + file_scanner := bufio.NewScanner(statFile) + + // Read CPU stats + for file_scanner.Scan() { + line := file_scanner.Text() + fields := strings.Fields(line) + if fields[0] == "cpu" { + for i := 1; i < len(fields); i++ { + val, err := strconv.ParseUint(fields[i], 10, 64) + if err != nil { + log.Panic("Error parsing CPU stats:", err) + } + + switch i { + case 1: + s.User = float64(val) + case 2: + s.Nice = float64(val) + case 3: + s.System = float64(val) + case 4: + s.Idle = float64(val) + case 5: + s.IOWait = float64(val) + } + } + break + } + } + + return s, nil +} + +// https://man7.org/linux/man-pages/man5/proc.5.html diff --git a/internal/stats/cpu/cpu_test.go b/internal/stats/cpu/cpu_test.go new file mode 100644 index 0000000..0135fb9 --- /dev/null +++ b/internal/stats/cpu/cpu_test.go @@ -0,0 +1,65 @@ +package cpu + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// Mock function +type MockFileOpener struct { + mock.Mock +} + +// Open opens a mock file for testing. +func (m *MockFileOpener) Open(name string) (*os.File, error) { + args := m.Called(name) + // Return a mocked file content for testing + return args.Get(0).(*os.File), args.Error(1) +} + +func TestJsonString(t *testing.T) { + cpuStats := &Stats{ + User: 10.5, + System: 0, + Kernel: 5.3, + Idle: 80.2, + IOWait: 3.0, + Nice: 0.1, + IdleTime: 200, + TotalCPUTime: 1000, + } + + expectedJson := `{"User":10.5,"System":0,"Kernel":5.3,"Idle":80.2,"IOWait":3,"Nice":0.1,"idleTime":200,"totalCPUTime":1000}` + jsonString := cpuStats.JsonString() + + if jsonString != expectedJson { + t.Errorf("JsonString() returned incorrect result. Expected: %s, Got: %s", expectedJson, jsonString) + } +} + +func TestNewStats(t *testing.T) { + mockFn := new(MockFileOpener) + testFile, tfErr := os.Open("./data/stat") + if tfErr != nil { + t.Error("Error getting /data/stat: ", tfErr) + } + mockFn.On("Open", "/proc/stat").Return(testFile, nil) + defer testFile.Close() + + stats, err := NewStats(mockFn) + stats.Compute() + + t.Run("OpenStatFile", func(t *testing.T) { + + assert.NoError(t, err) + mockFn.AssertCalled(t, "Open", "/proc/stat") + if assert.NotNil(t, stats) { + assert.Equal(t, float64(851951), stats.User) + assert.Equal(t, float64(710694), stats.System) + } + stats.JsonString() + }) +} diff --git a/internal/stats/cpu/data/stat b/internal/stats/cpu/data/stat new file mode 100644 index 0000000..ea7ef64 --- /dev/null +++ b/internal/stats/cpu/data/stat @@ -0,0 +1,20 @@ +cpu 851951 987 710694 143031281 262586 0 161886 0 0 0 +cpu0 82419 0 72614 11870026 30073 0 87097 0 0 0 +cpu1 61762 1 47971 11944628 26996 0 31627 0 0 0 +cpu2 82249 85 70782 11876269 28702 0 11024 0 0 0 +cpu3 59662 0 47531 11959583 17182 0 5542 0 0 0 +cpu4 81500 7 70085 11883142 26100 0 4226 0 0 0 +cpu5 59555 21 48009 11957626 19125 0 3383 0 0 0 +cpu6 80812 87 69589 11884945 24162 0 3578 0 0 0 +cpu7 61317 395 47945 11958416 15944 0 2870 0 0 0 +cpu8 80678 10 70166 11890185 19868 0 3509 0 0 0 +cpu9 59709 0 47812 11962195 15192 0 2803 0 0 0 +cpu10 82024 254 70358 11885165 21629 0 3494 0 0 0 +cpu11 60264 127 47832 11959097 17608 0 2733 0 0 0 +intr 137123267 0 0 0 0 0 0 0 0 0 45 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1830 1 4 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +ctxt 646731886 +btime 1712177864 +processes 550084 +procs_running 1 +procs_blocked 0 +softirq 81792754 0 9819540 19 15860854 4390 0 2896918 28715855 33 24495145 \ No newline at end of file diff --git a/internal/stats/utils/io.go b/internal/stats/utils/io.go new file mode 100644 index 0000000..a248973 --- /dev/null +++ b/internal/stats/utils/io.go @@ -0,0 +1,15 @@ +package utils + +import "os" + +// Interface for opening files. +type FileOpener interface { + Open(name string) (*os.File, error) +} + +type StatFileOpener struct{} + +// opens using os.Open as O_RDONLY +func (o *StatFileOpener) Open(name string) (*os.File, error) { + return os.Open(name) +} From 2aaf70391a02c3416ec7c32df14e802c54b79459 Mon Sep 17 00:00:00 2001 From: Egbewatt Kokou Mwangah Smith Date: Tue, 7 May 2024 18:46:33 -0600 Subject: [PATCH 5/5] get system stats --- internal/stats/system/data/version | 1 + internal/stats/system/system.go | 64 ++++++++++++++++++++++++++++ internal/stats/system/system_test.go | 60 ++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 internal/stats/system/data/version create mode 100644 internal/stats/system/system.go create mode 100644 internal/stats/system/system_test.go diff --git a/internal/stats/system/data/version b/internal/stats/system/data/version new file mode 100644 index 0000000..dd35555 --- /dev/null +++ b/internal/stats/system/data/version @@ -0,0 +1 @@ +Linux version 5.4.0-97-generic (buildd@lgw01-amd64-023) (gcc version 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04)) #110-Ubuntu SMP Tue Sep 28 20:17:47 UTC 2021 \ No newline at end of file diff --git a/internal/stats/system/system.go b/internal/stats/system/system.go new file mode 100644 index 0000000..70f4005 --- /dev/null +++ b/internal/stats/system/system.go @@ -0,0 +1,64 @@ +package system + +import ( + "bufio" + "encoding/json" + "log" + "strings" + "taskweaver/internal/stats/utils" +) + +// System Version Info (Linux kernel version , the build host , the compiler version, and the build date ) +type Stats struct { + OSVersion string `json:"OSVersion"` + KernelVersion string `json:"KernelVersion"` + BuildInfo string `json:"BuildInfo"` +} + +// serialized output +func (system_stats *Stats) JsonString() string { + // + jsonData, err := json.Marshal(system_stats) + if err != nil { + log.Panic("Error:", err) + } + return string(jsonData) +} + +func NewStats(fileOpener utils.FileOpener) (*Stats, error) { + // compute stats + stats := &Stats{} + + // stat file + versionFile, err := fileOpener.Open("/proc/version") + if err != nil { + log.Panic("Error opening /proc/version file: ", err) + } + defer versionFile.Close() + + // File Scanner + file_scanner := bufio.NewScanner(versionFile) + + // Read System stats + for file_scanner.Scan() { + line := file_scanner.Text() + fields := strings.Fields(line) + if fields[0] == "Linux" { + for i := 0; i < len(fields); i++ { + switch i { + case 0: + stats.OSVersion = fields[i] + case 2: + stats.KernelVersion = fields[i] + case 3: + stats.BuildInfo = fields[i] + } + } + break + } + } + + return stats, nil +} + +// https://man7.org/linux/man-pages/man5/proc.5.html diff --git a/internal/stats/system/system_test.go b/internal/stats/system/system_test.go new file mode 100644 index 0000000..a260762 --- /dev/null +++ b/internal/stats/system/system_test.go @@ -0,0 +1,60 @@ +package system + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// Mock function +type MockFileOpener struct { + mock.Mock +} + +// Open opens a mock file for testing. +func (m *MockFileOpener) Open(name string) (*os.File, error) { + args := m.Called(name) + // Return a mocked file content for testing + return args.Get(0).(*os.File), args.Error(1) +} + +func TestJsonString(t *testing.T) { + cpuStats := &Stats{ + OSVersion: "Linux", + KernelVersion: "5.4.0-97-generic", + BuildInfo: "(buildd@lgw01-amd64-023)", + } + + expectedJson := `{"OSVersion":"Linux","KernelVersion":"5.4.0-97-generic","BuildInfo":"(buildd@lgw01-amd64-023)"}` + jsonString := cpuStats.JsonString() + + if jsonString != expectedJson { + t.Errorf("JsonString() returned incorrect result. Expected: %s, Got: %s", expectedJson, jsonString) + } +} + +func TestNewStats(t *testing.T) { + mockFn := new(MockFileOpener) + testFile, tfErr := os.Open("./data/version") + if tfErr != nil { + t.Error("Error getting /data/version: ", tfErr) + } + mockFn.On("Open", "/proc/version").Return(testFile, nil) + defer testFile.Close() + + stats, err := NewStats(mockFn) + + t.Run("OpenStatFile", func(t *testing.T) { + + assert.NoError(t, err) + mockFn.AssertCalled(t, "Open", "/proc/version") + if assert.NotNil(t, stats) { + assert.Equal(t, "Linux", stats.OSVersion) + assert.Equal(t, "5.4.0-97-generic", stats.KernelVersion) + assert.Equal(t, "(buildd@lgw01-amd64-023)", stats.BuildInfo) + } + stats.JsonString() + }) +}