nodes flow

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+
+

.github/workflows/build.yml

@@ -0,0 +1,36 @@
+name: Build & Tests
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+
+env:
+  GO_VERSION: 1.21.5
+
+jobs:
+  project_build:
+    name: TaskWeaver Build
+    permissions:
+      contents: read
+      issues: read
+      checks: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      -  name: Set up Go ${{ env.GO_VERSION }}
+         uses: actions/setup-go@v4
+         with:
+           go-version: ${{ env.GO_VERSION }}
+           cache-dependency-path: go.sum 
+
+      - name: Display Go version
+        run: go version
+
+      - name: Build & Test
+        run: |
+          chmod +x ci/scripts/go-build-test.sh
+          ci/scripts/go-build-test.sh

.github/workflows/codeql.yaml

@@ -0,0 +1,34 @@
+name: "CodeQL"
+
+on:
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: '0 3 * * */2'
+
+permissions:
+  security-events: write
+  actions: read
+  contents: read
+
+jobs:
+  analyse:
+    name: Analyse
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: go
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:go"

README.md

@@ -2,19 +2,6 @@
 
 TaskWeaver is a comprehensive platform designed to streamline task management, automate repetitive processes, and enhance overall system efficiency. By providing users with a user-friendly interface and advanced scheduling capabilities, this project empowers Platform Engineering teams to optimize their workflows and reduce manual intervention.
 
-## Table of Contents
-
-- [Introduction](#introduction)
-- [Features](#features)
-- [Architecture](#architecture)
-- [Plugin System](#plugin-system)
-- [Integration with External Systems](#integration-with-external-systems)
-- [Research](#research)
-
-## Introduction
-
-Task Automation and Scheduling is a critical component of modern Platform Engineering. This project aims to offer a robust solution for defining, scheduling, and executing various tasks, enabling users to automate routine activities, minimize errors, and enhance productivity. From managing simple recurring tasks to orchestrating complex workflows, this platform provides the tools necessary to achieve these goals effectively.
-
 ## Features
 
 **Task Definition and Management:** Users can easily define tasks through an intuitive interface or a configuration file. Each task includes execution commands, input parameters, and dependency specifications. The platform allows users to create, update, delete, and monitor their scheduled jobs.
@@ -37,20 +24,10 @@ Task Automation and Scheduling is a critical component of modern Platform Engine
 
 **Integration with External Systems:** Integration with version control systems (e.g., Git) automates task triggering upon specific branch changes. Additionally, collaboration with cloud providers and container orchestrators maximizes resource utilization.
 
-## Architecture
-
-The architecture of this platform revolves around distributed job scheduling. It leverages a combination of single-node and cluster setups to efficiently manage tasks. This architecture accommodates various platforms and environments, making it adaptable to Windows, Linux, and macOS operating systems.
-
-[https://lucid.app/lucidspark/3b091858-07f0-4443-ad49-5b87a11b78e0/edit?invitationId=inv_e966ce87-b4b7-4572-962f-bf8879dee5e3](https://lucid.app/lucidspark/3b091858-07f0-4443-ad49-5b87a11b78e0/edit?invitationId=inv_e966ce87-b4b7-4572-962f-bf8879dee5e3)
-
 ## Plugin System
 
 A plugin system enhances the project's extensibility and versatility. It introduces a well-defined plugin interface, a registry for plugin management, dynamic loading capabilities, and user-configurable plugins. This system enables users to define custom task types, integrate with external systems, enhance scheduling options, and expand notification channels.
 
-## Integration with External Systems
-
-The project integrates seamlessly with external systems, such as Kubernetes (K8s), Apache Kafka, Apache Airflow, and AWS Batch. These integrations provide users with additional options for managing tasks, enhancing scalability, and leveraging industry-standard tools.
-
 ## Research
 
 The development of this project involved research and inspiration from various existing systems:

ci/scripts/go-build-test.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -eu
+
+USAGE="USAGE:
+${0}"
+
+if [[ $# -ne 0 ]]; then
+    echo "${USAGE}" >&2
+    exit 1
+fi
+
+go build -v ./...
+go test -v ./...

internal/stats/cpu/cpu.go

@@ -0,0 +1,95 @@
+package cpu
+
+import (
+	"bufio"
+	"encoding/json"
+	"log"
+	"strconv"
+	"strings"
+	"taskweaver/internal/stats/utils"
+)
+
+const (
+	// stat file path
+	filePath = "/proc/stat"
+)
+
+// CPU Stats
+type Stats struct {
+	// Time spent in user mode
+	User float64 `json:"User"`
+	// Time spent in system mode
+	System float64 `json:"System"`
+	Kernel float64 `json:"Kernel"`
+	// Time spent in the idle task
+	Idle float64 `json:"Idle"`
+	// Time waiting for I/O to complete.
+	IOWait float64 `json:"IOWait"`
+	// Time spent in user mode with low priority
+	Nice         float64 `json:"Nice"`
+	IdleTime     uint64  `json:"idleTime"`
+	TotalCPUTime uint64  `json:"totalCPUTime"`
+	fileOpener   utils.FileOpener
+}
+
+// serialized output
+func (cpu_stats *Stats) JsonString() string {
+	//
+	jsonData, err := json.Marshal(cpu_stats)
+	if err != nil {
+		log.Panic("Error:", err)
+	}
+	return string(jsonData)
+}
+
+// FileOpener is needed for testing purposes and also coordinate permissions for file access
+func NewStats(fileOpener utils.FileOpener) (*Stats, error) {
+	return &Stats{
+		fileOpener: fileOpener,
+	}, nil
+}
+
+func (s *Stats) Compute() (*Stats, error) {
+	// compute stats
+
+	statFile, err := s.fileOpener.Open(filePath)
+	if err != nil {
+		log.Panicf(`Error opening %s file: %e`, filePath, err)
+	}
+	defer statFile.Close()
+
+	// File Scanner
+	file_scanner := bufio.NewScanner(statFile)
+
+	// Read CPU stats
+	for file_scanner.Scan() {
+		line := file_scanner.Text()
+		fields := strings.Fields(line)
+		if fields[0] == "cpu" {
+			for i := 1; i < len(fields); i++ {
+				val, err := strconv.ParseUint(fields[i], 10, 64)
+				if err != nil {
+					log.Panic("Error parsing CPU stats:", err)
+				}
+
+				switch i {
+				case 1:
+					s.User = float64(val)
+				case 2:
+					s.Nice = float64(val)
+				case 3:
+					s.System = float64(val)
+				case 4:
+					s.Idle = float64(val)
+				case 5:
+					s.IOWait = float64(val)
+				}
+			}
+			break
+		}
+	}
+
+	return s, nil
+}
+
+// https://man7.org/linux/man-pages/man5/proc.5.html

internal/stats/cpu/cpu_test.go

@@ -0,0 +1,65 @@
+package cpu
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+// Mock function
+type MockFileOpener struct {
+	mock.Mock
+}
+
+// Open opens a mock file for testing.
+func (m *MockFileOpener) Open(name string) (*os.File, error) {
+	args := m.Called(name)
+	// Return a mocked file content for testing
+	return args.Get(0).(*os.File), args.Error(1)
+}
+
+func TestJsonString(t *testing.T) {
+	cpuStats := &Stats{
+		User:         10.5,
+		System:       0,
+		Kernel:       5.3,
+		Idle:         80.2,
+		IOWait:       3.0,
+		Nice:         0.1,
+		IdleTime:     200,
+		TotalCPUTime: 1000,
+	}
+
+	expectedJson := `{"User":10.5,"System":0,"Kernel":5.3,"Idle":80.2,"IOWait":3,"Nice":0.1,"idleTime":200,"totalCPUTime":1000}`
+	jsonString := cpuStats.JsonString()
+
+	if jsonString != expectedJson {
+		t.Errorf("JsonString() returned incorrect result. Expected: %s, Got: %s", expectedJson, jsonString)
+	}
+}
+
+func TestNewStats(t *testing.T) {
+	mockFn := new(MockFileOpener)
+	testFile, tfErr := os.Open("./data/stat")
+	if tfErr != nil {
+		t.Error("Error getting /data/stat: ", tfErr)
+	}
+	mockFn.On("Open", "/proc/stat").Return(testFile, nil)
+	defer testFile.Close()
+
+	stats, err := NewStats(mockFn)
+	stats.Compute()
+
+	t.Run("OpenStatFile", func(t *testing.T) {
+
+		assert.NoError(t, err)
+		mockFn.AssertCalled(t, "Open", "/proc/stat")
+		if assert.NotNil(t, stats) {
+			assert.Equal(t, float64(851951), stats.User)
+			assert.Equal(t, float64(710694), stats.System)
+		}
+		stats.JsonString()
+	})
+}

internal/stats/cpu/data/stat

@@ -0,0 +1,20 @@
+cpu  851951 987 710694 143031281 262586 0 161886 0 0 0
+cpu0 82419 0 72614 11870026 30073 0 87097 0 0 0
+cpu1 61762 1 47971 11944628 26996 0 31627 0 0 0
+cpu2 82249 85 70782 11876269 28702 0 11024 0 0 0
+cpu3 59662 0 47531 11959583 17182 0 5542 0 0 0
+cpu4 81500 7 70085 11883142 26100 0 4226 0 0 0
+cpu5 59555 21 48009 11957626 19125 0 3383 0 0 0
+cpu6 80812 87 69589 11884945 24162 0 3578 0 0 0
+cpu7 61317 395 47945 11958416 15944 0 2870 0 0 0
+cpu8 80678 10 70166 11890185 19868 0 3509 0 0 0
+cpu9 59709 0 47812 11962195 15192 0 2803 0 0 0
+cpu10 82024 254 70358 11885165 21629 0 3494 0 0 0
+cpu11 60264 127 47832 11959097 17608 0 2733 0 0 0
+intr 137123267 0 0 0 0 0 0 0 0 0 45 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1830 1 4 31 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ctxt 646731886
+btime 1712177864
+processes 550084
+procs_running 1
+procs_blocked 0
+softirq 81792754 0 9819540 19 15860854 4390 0 2896918 28715855 33 24495145

internal/stats/system/data/version

@@ -0,0 +1 @@
+Linux version 5.4.0-97-generic (buildd@lgw01-amd64-023) (gcc version 9.3.0 (Ubuntu 9.3.0-17ubuntu1~20.04)) #110-Ubuntu SMP Tue Sep 28 20:17:47 UTC 2021

internal/stats/system/system.go

@@ -0,0 +1,64 @@
+package system
+
+import (
+	"bufio"
+	"encoding/json"
+	"log"
+	"strings"
+	"taskweaver/internal/stats/utils"
+)
+
+// System Version Info (Linux kernel version , the build host , the compiler version, and the build date )
+type Stats struct {
+	OSVersion     string `json:"OSVersion"`
+	KernelVersion string `json:"KernelVersion"`
+	BuildInfo     string `json:"BuildInfo"`
+}
+
+// serialized output
+func (system_stats *Stats) JsonString() string {
+	//
+	jsonData, err := json.Marshal(system_stats)
+	if err != nil {
+		log.Panic("Error:", err)
+	}
+	return string(jsonData)
+}
+
+func NewStats(fileOpener utils.FileOpener) (*Stats, error) {
+	// compute stats
+	stats := &Stats{}
+
+	// stat file
+	versionFile, err := fileOpener.Open("/proc/version")
+	if err != nil {
+		log.Panic("Error opening /proc/version file: ", err)
+	}
+	defer versionFile.Close()
+
+	// File Scanner
+	file_scanner := bufio.NewScanner(versionFile)
+
+	// Read System stats
+	for file_scanner.Scan() {
+		line := file_scanner.Text()
+		fields := strings.Fields(line)
+		if fields[0] == "Linux" {
+			for i := 0; i < len(fields); i++ {
+				switch i {
+				case 0:
+					stats.OSVersion = fields[i]
+				case 2:
+					stats.KernelVersion = fields[i]
+				case 3:
+					stats.BuildInfo = fields[i]
+				}
+			}
+			break
+		}
+	}
+
+	return stats, nil
+}
+
+// https://man7.org/linux/man-pages/man5/proc.5.html