commit 3309a47e10e791153ae7dfeff7b86bad9ae0df8f Author: bachko Date: Thu Jul 9 13:52:31 2026 +0000 Initial commit: SoS bot framework - ADB client (connect, tap, swipe, screencap, app control) - Vision module (in-memory screenshot + OpenCV template matching) - Action loader (auto-discovers .py files in actions/) - Engine (screenshot → detect → act loop with YAML playbook) - 12 built-in actions: tap, click, wait, swipe, find, find_and_click, wait_until, click_seq, keyevent, app, log - Example custom action template - Debug screenshot tool for template creation - Detailed plan.md with 20 actions across 9 phases - Sample playbook with 10 steps diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9957d4a --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ + +# Virtual environments +.venv/ +venv/ + +# OS +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ + +# Screenshots (user-generated debug files) +sos_screenshot_*.png + +# Template work files +templates/*.xcf +templates/*.psd + +# Playbook backups +playbook/*.bak* +playbook/*.old* diff --git a/README.md b/README.md new file mode 100644 index 0000000..a5d5c7c --- /dev/null +++ b/README.md @@ -0,0 +1,143 @@ +# State of Survival Bot + +ADB + OpenCV Python bot for State of Survival running in BlueStacks. + +## Quick Start + +```bash +# 1. Activate venv +cd sos_bot +source .venv/bin/activate + +# 2. Verify ADB connects to BlueStacks +adb connect 127.0.0.1:5555 +adb devices + +# 3. Capture the first screenshot +python debug_screenshot.py + +# 4. Crop reference templates from the screenshot, save to templates/ +# (use GIMP, Krita, MSPaint, or any image editor) + +# 5. Edit playbook/default.yaml to add steps matching your templates + +# 6. Run the bot +python main.py +``` + +## Architecture + +``` +sos_bot/ +├── main.py # Entry point +├── debug_screenshot.py # Capture screenshots for template creation +├── requirements.txt +├── core/ +│ ├── adb.py # ADB client (connect, tap, swipe, screencap) +│ ├── vision.py # In-memory screenshot + OpenCV template matching +│ ├── engine.py # Main loop: screenshot → detect → act +│ └── action_loader.py # Dynamic action discovery (.py files) +├── actions/ +│ ├── tap.py # Tap at (x, y) +│ ├── click.py # Alias for tap +│ ├── wait.py # Sleep N seconds +│ ├── swipe.py # Swipe gesture +│ ├── find.py # Locate a template (no click) +│ ├── find_and_click.py # Locate template → tap it +│ ├── wait_until.py # Wait for template to appear +│ ├── click_seq.py # Tap multiple coordinates in sequence +│ ├── keyevent.py # Android key events (back, home, etc.) +│ ├── app.py # Launch/stop/restart apps +│ └── log.py # Write to log +├── templates/ # Reference PNG images for matching +│ └── README.txt +└── playbook/ + └── default.yaml # Step-by-step behavior definition +``` + +## How It Works + +1. **Main loop** takes a screenshot every `loop_delay` seconds +2. Each playbook **step** checks its condition against the screenshot +3. The **first matching step** runs its action sequence +4. Screenshots are **never saved to disk** — they live in memory as numpy arrays + +## Playbook Conditions + +Each step can use one of these triggers: + +| Condition | Description | +|-----------|-------------| +| `if_template` | Template matching — looks for a reference PNG | +| `if_pixel` | Pixel color check at specific coordinates | +| `if_multiple` | All specified templates must match (AND) | +| `if_counter` | Triggers every N iterations | +| `always: true` | Always matches (fallback / idle) | + +## Playbook Actions + +| Action | Params | What it does | +|--------|--------|-------------| +| `tap` | `x`, `y` | Tap at coordinates | +| `click` | `x`, `y` | Same as tap | +| `wait` | `seconds` | Pause N seconds | +| `find` | `template`, `confidence`, `region` | Find template (returns position) | +| `find_and_click` | `template`, `confidence`, `region`, `offset_x/y` | Find + tap | +| `wait_until` | `template`, `confidence`, `timeout`, `poll_interval` | Wait for template | +| `swipe` | `x1`, `y1`, `x2`, `y2`, `duration_ms` | Swipe gesture | +| `click_seq` | `coords` (list of [x,y]), `delay` | Tap sequence | +| `keyevent` | `key` (back/home/menu/enter) | Android key event | +| `app` | `action` (launch/stop/restart), `package` | App control | +| `log` | `message` | Write to log | + +## Extending: Custom Actions + +Create a new `.py` file in `actions/` with this structure: + +```python +"""Action: my_action — description of what it does.""" + + +class Action: + name = "my_action" + + def run(self, ctx, params): + # ctx = {adb, vision, engine, screenshot, log, _templates_dir} + # params = dict from playbook YAML + + x = params["x"] + y = params["y"] + ctx["adb"].tap(x, y) + ctx["log"].info("my_action @ (%d, %d)", x, y) +``` + +The file is auto-discovered on next startup. No registration needed. + +## Creating Templates + +1. Run `python debug_screenshot.py` → saves current screen +2. Open the PNG in an image editor +3. Crop EXACTLY the button/icon you want to detect +4. Save as `templates/my_button.png` +5. Reference it in the playbook: `if_template: "my_button.png"` + +**Template tips:** +- Crop tightly around the target (no excess background) +- Use a unique, static UI element +- Start with `confidence: 0.8`, lower to 0.7 if it's not found +- Use `region: [x1, y1, x2, y2]` to constrain search area (faster, fewer false positives) + +## Common ADB Ports + +| BlueStacks Instance | Port | +|-------------------|------| +| First instance | 5555 | +| Second instance | 5557 | +| Third instance | 5559 | +| (Nth) | 5555 + (N-1)*2 | + +## BlueStacks Settings Required + +- Settings → Advanced → Android Debug Bridge (ADB): **ON** +- Settings → Display → Resolution: 1600×900 (recommended) +- Settings → Display → DPI: 240 diff --git a/actions/__init__.py b/actions/__init__.py new file mode 100644 index 0000000..a0a65fa --- /dev/null +++ b/actions/__init__.py @@ -0,0 +1 @@ +"""Actions __init__ — this directory holds pluggable action modules.""" diff --git a/actions/app.py b/actions/app.py new file mode 100644 index 0000000..98495b4 --- /dev/null +++ b/actions/app.py @@ -0,0 +1,24 @@ +"""Action: app — launch or force-stop an app.""" +import time + + +class Action: + name = "app" + + def run(self, ctx, params): + action = params.get("action", "launch") + package = params.get("package", "com.kingsgroup.sos") + + if action == "launch": + ctx["adb"].launch_app(package) + ctx["log"].info("Launched %s", package) + time.sleep(params.get("wait", 5)) + elif action == "stop": + ctx["adb"].force_stop(package) + ctx["log"].info("Stopped %s", package) + elif action == "restart": + ctx["adb"].force_stop(package) + time.sleep(2) + ctx["adb"].launch_app(package) + ctx["log"].info("Restarted %s", package) + time.sleep(params.get("wait", 10)) diff --git a/actions/click.py b/actions/click.py new file mode 100644 index 0000000..4060915 --- /dev/null +++ b/actions/click.py @@ -0,0 +1,12 @@ +"""Action: click — alias for tap.""" +import time + + +class Action: + name = "click" + + def run(self, ctx, params): + x = params["x"] + y = params["y"] + ctx["adb"].tap(x, y) + ctx["log"].info("click(%d, %d)", x, y) diff --git a/actions/click_seq.py b/actions/click_seq.py new file mode 100644 index 0000000..4ec6194 --- /dev/null +++ b/actions/click_seq.py @@ -0,0 +1,19 @@ +"""Action: click_seq — tap a sequence of coordinates with optional delay.""" +import time + + +class Action: + name = "click_seq" + + def run(self, ctx, params): + points = params.get("points", params.get("coords", [])) + delay = params.get("delay", 0.5) + + for pt in points: + if isinstance(pt, (list, tuple)): + x, y = pt[0], pt[1] + else: + continue + ctx["adb"].tap(x, y) + ctx["log"].info("click_seq: tap(%d, %d)", x, y) + time.sleep(delay) diff --git a/actions/example_action.py b/actions/example_action.py new file mode 100644 index 0000000..185aa0c --- /dev/null +++ b/actions/example_action.py @@ -0,0 +1,56 @@ +""" +Action: my_custom_action — TEMPLATE for creating new actions. +Copy this file to actions/my_custom_action.py and customize. + +Requirements: + - Class must be named 'Action' + - Must have a unique 'name' attribute + - Must have a 'run(self, ctx, params)' method +""" +import time + + +class Action: + name = "my_custom_action" + + def run(self, ctx, params): + """ + ctx provides: + ctx["adb"] — AdbClient (tap, swipe, keyevent, text, etc.) + ctx["vision"] — Vision (find_template, pixel_color, etc.) + ctx["engine"] — Engine (stop(), config) + ctx["screenshot"] — current screenshot (numpy BGR array) + ctx["log"] — logger (.info, .warning, .error) + ctx["_templates_dir"] — path to templates/ directory + + params contains whatever the playbook passes, e.g.: + params.get("x") + params.get("y") + params.get("message") + params.get("my_param", "default_value") + """ + + # Example: read params and do something + x = params.get("x", 100) + y = params.get("y", 100) + msg = params.get("message", "Custom action ran") + + # Log it + ctx["log"].info("[%s] %s (x=%d, y=%d)", self.name, msg, x, y) + + # Do something with ADB + ctx["adb"].tap(x, y) + + # Wait + time.sleep(params.get("delay", 1.0)) + + # You can also do template matching + # from pathlib import Path + # import os + # tmpl = os.path.join(ctx["_templates_dir"], "my_template.png") + # pos = ctx["vision"].find_template(tmpl, confidence=0.8) + # if pos: + # ctx["adb"].tap(pos[0], pos[1]) + + # Return value is ignored; set ctx keys for other actions to read + ctx["_last_result"] = f"Done: {msg}" diff --git a/actions/find.py b/actions/find.py new file mode 100644 index 0000000..a554629 --- /dev/null +++ b/actions/find.py @@ -0,0 +1,23 @@ +"""Action: find — find a template and return its position (sets ctx['_last_pos']).""" +import os + + +class Action: + name = "find" + + def run(self, ctx, params): + template = params["template"] + confidence = params.get("confidence", 0.8) + region = params.get("region") + + tmpl_dir = ctx.get("_templates_dir", + os.path.join(os.path.dirname(__file__), "..", "templates")) + tpl_path = os.path.join(tmpl_dir, template) + + pos = ctx["vision"].find_template(tpl_path, confidence, region) + ctx["_last_pos"] = pos + if pos is None: + ctx["log"].info("find(%s) → not found", template) + else: + ctx["log"].info("find(%s) → %s", template, pos) + return pos diff --git a/actions/find_and_click.py b/actions/find_and_click.py new file mode 100644 index 0000000..04c5dea --- /dev/null +++ b/actions/find_and_click.py @@ -0,0 +1,27 @@ +"""Action: find_and_click — locate a template image and tap it.""" +import os + + +class Action: + name = "find_and_click" + + def run(self, ctx, params): + template = params["template"] + confidence = params.get("confidence", 0.8) + region = params.get("region") + offset_x = params.get("offset_x", 0) + offset_y = params.get("offset_y", 0) + + tmpl_dir = ctx.get("_templates_dir", + os.path.join(os.path.dirname(__file__), "..", "templates")) + tpl_path = os.path.join(tmpl_dir, template) + + pos = ctx["vision"].find_template(tpl_path, confidence, region) + if pos is None: + ctx["log"].info("find_and_click(%s) → not found, skipping", template) + return None + + cx, cy = pos[0] + offset_x, pos[1] + offset_y + ctx["adb"].tap(cx, cy) + ctx["log"].info("find_and_click(%s) → tap(%d, %d)", template, cx, cy) + return (cx, cy) diff --git a/actions/keyevent.py b/actions/keyevent.py new file mode 100644 index 0000000..a45121e --- /dev/null +++ b/actions/keyevent.py @@ -0,0 +1,21 @@ +"""Action: keyevent — send Android key event.""" + + +class Action: + name = "keyevent" + + KEYS = { + "back": 4, + "home": 3, + "menu": 82, + "enter": 66, + "space": 62, + "esc": 111, + } + + def run(self, ctx, params): + key = params.get("key", params.get("keycode", "back")) + if isinstance(key, str): + key = self.KEYS.get(key.lower(), 4) + ctx["adb"].keyevent(key) + ctx["log"].info("keyevent(%d)", key) diff --git a/actions/log.py b/actions/log.py new file mode 100644 index 0000000..cf51672 --- /dev/null +++ b/actions/log.py @@ -0,0 +1,10 @@ +"""Action: log — write a message to the bot log.""" +import time + + +class Action: + name = "log" + + def run(self, ctx, params): + msg = params.get("message", params.get("msg", "")) + ctx["log"].info("[PLAYBOOK] %s", msg) diff --git a/actions/swipe.py b/actions/swipe.py new file mode 100644 index 0000000..8c38996 --- /dev/null +++ b/actions/swipe.py @@ -0,0 +1,15 @@ +"""Action: swipe — perform a swipe gesture.""" + + +class Action: + name = "swipe" + + def run(self, ctx, params): + x1 = params["x1"] + y1 = params["y1"] + x2 = params["x2"] + y2 = params["y2"] + duration = params.get("duration_ms", 300) + + ctx["adb"].swipe(x1, y1, x2, y2, duration) + ctx["log"].info("swipe(%d,%d → %d,%d, %dms)", x1, y1, x2, y2, duration) diff --git a/actions/tap.py b/actions/tap.py new file mode 100644 index 0000000..15158b7 --- /dev/null +++ b/actions/tap.py @@ -0,0 +1,12 @@ +"""Action: tap — tap at screen coordinates.""" +import time + + +class Action: + name = "tap" + + def run(self, ctx, params): + x = params["x"] + y = params["y"] + ctx["adb"].tap(x, y) + ctx["log"].info("tap(%d, %d)", x, y) diff --git a/actions/wait.py b/actions/wait.py new file mode 100644 index 0000000..832dedf --- /dev/null +++ b/actions/wait.py @@ -0,0 +1,11 @@ +"""Action: wait — pause for N seconds.""" +import time + + +class Action: + name = "wait" + + def run(self, ctx, params): + seconds = params.get("seconds", params.get("ms", 1000) / 1000) + ctx["log"].info("wait(%.1fs)", seconds) + time.sleep(seconds) diff --git a/actions/wait_until.py b/actions/wait_until.py new file mode 100644 index 0000000..73eb509 --- /dev/null +++ b/actions/wait_until.py @@ -0,0 +1,29 @@ +"""Action: wait_until — wait for a template to appear (with timeout).""" +import os +import time + + +class Action: + name = "wait_until" + + def run(self, ctx, params): + template = params["template"] + confidence = params.get("confidence", 0.8) + timeout = params.get("timeout", 30) + poll = params.get("poll_interval", 1) + + tmpl_dir = ctx.get("_templates_dir", + os.path.join(os.path.dirname(__file__), "..", "templates")) + tpl_path = os.path.join(tmpl_dir, template) + + deadline = time.time() + timeout + while time.time() < deadline: + ctx["vision"].capture() + pos = ctx["vision"].find_template(tpl_path, confidence) + if pos is not None: + ctx["log"].info("wait_until(%s) → found at %s", template, pos) + return pos + time.sleep(poll) + + ctx["log"].warning("wait_until(%s) → timeout after %.0fs", template, timeout) + return None diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/action_loader.py b/core/action_loader.py new file mode 100644 index 0000000..7c0c9fc --- /dev/null +++ b/core/action_loader.py @@ -0,0 +1,79 @@ +""" +Action Loader — dynamically discovers and loads action modules +from the actions/ directory. Each .py file is a self-contained action. +""" +import importlib.util +import logging +import os +import sys + +logger = logging.getLogger("sos.actions") + + +class ActionRegistry: + """Holds all registered actions by name.""" + + def __init__(self): + self._actions = {} + + def register(self, action): + """Register an action instance.""" + self._actions[action.name] = action + + def get(self, name: str): + """Get an action by name, or None.""" + return self._actions.get(name) + + def list(self): + """Return all registered action names.""" + return list(self._actions.keys()) + + def run(self, name: str, ctx: dict, params: dict): + """Run an action by name. Raises KeyError if not found.""" + action = self.get(name) + if action is None: + raise KeyError(f"Unknown action: '{name}' (available: {self.list()})") + return action.run(ctx, params) + + +def load_actions(actions_dir: str) -> ActionRegistry: + """ + Scan actions_dir for .py files (except __init__.py) and load each. + Each module must export a class named 'Action' with: + - name: str (the playbook-visible name) + - def run(self, ctx: dict, params: dict) -> any + Returns an ActionRegistry with all loaded actions. + """ + registry = ActionRegistry() + if not os.path.isdir(actions_dir): + logger.warning("Actions directory not found: %s", actions_dir) + return registry + + sys.path.insert(0, os.path.dirname(actions_dir)) + + for fname in sorted(os.listdir(actions_dir)): + if not fname.endswith(".py") or fname == "__init__.py": + continue + mod_path = os.path.join(actions_dir, fname) + mod_name = f"sos_actions_{fname[:-3]}" + + spec = importlib.util.spec_from_file_location(mod_name, mod_path) + if spec is None or spec.loader is None: + logger.warning("Cannot load action module: %s", fname) + continue + + try: + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + action_class = getattr(mod, "Action", None) + if action_class is None: + logger.warning("Action %s has no 'Action' class — skipping", fname) + continue + instance = action_class() + registry.register(instance) + logger.info("Loaded action: %s (%s)", instance.name, fname) + except Exception as e: + logger.warning("Failed to load action %s: %s", fname, e) + + sys.path.pop(0) + return registry diff --git a/core/adb.py b/core/adb.py new file mode 100644 index 0000000..b338cac --- /dev/null +++ b/core/adb.py @@ -0,0 +1,119 @@ +""" +ADB client — communicates with BlueStacks emulator via Android Debug Bridge. +All screenshot data stays in memory (never written to disk). +""" +import subprocess +import time +import logging + +logger = logging.getLogger("sos.adb") + + +class AdbError(Exception): + """Raised when an ADB command fails.""" + pass + + +class AdbClient: + """Low-level ADB wrapper over a single device.""" + + def __init__(self, host="127.0.0.1", port=5555): + self.host = host + self.port = port + self.serial = None # set by connect() + + # ── connection ──────────────────────────────────────────── + + def connect(self) -> str: + """Connect to the emulator. Returns device serial.""" + out = self._raw(f"connect {self.host}:{self.port}") + self.serial = f"{self.host}:{self.port}" + lines = [l.strip() for l in out.splitlines() if l.strip()] + logger.info("ADB connect: %s", lines[-1] if lines else out) + return self.serial + + def disconnect(self): + """Disconnect from the emulator.""" + if self.serial: + self._raw(f"disconnect {self.serial}") + + def is_connected(self) -> bool: + """Check if device shows as 'device' in adb devices.""" + try: + out = self._raw("devices") + return self.serial in out and "device" in out.split(self.serial)[1][:20] + except Exception: + return False + + # ── screenshots (pure memory — no files) ────────────────── + + def screencap(self) -> bytes: + """ + Capture screen as raw PNG bytes, fully in memory. + Returns PNG bytes ready for OpenCV / PIL. + """ + raw = self._exec_out(f"shell screencap -p") + if not raw: + raise AdbError("screencap returned empty bytes — device disconnected?") + return raw + + # ── input actions ───────────────────────────────────────── + + def tap(self, x: int, y: int): + """Tap at screen coordinates (x, y).""" + self._shell(f"input tap {x} {y}") + + def swipe(self, x1: int, y1: int, x2: int, y2: int, duration_ms: int = 300): + """Swipe from (x1,y1) to (x2,y2) over duration_ms.""" + self._shell(f"input swipe {x1} {y1} {x2} {y2} {duration_ms}") + + def keyevent(self, keycode: int): + """Send a key event (e.g. 4 = BACK, 3 = HOME).""" + self._shell(f"input keyevent {keycode}") + + def text(self, text: str): + """Type text (only ASCII safe; use with care).""" + # escape special chars for shell + safe = text.replace(" ", "%s").replace("'", "\\'") + self._shell(f"input text '{safe}'") + + # ── app management ──────────────────────────────────────── + + def force_stop(self, package: str): + """Force-stop an app by package name.""" + self._shell(f"am force-stop {package}") + + def launch_app(self, package: str, activity: str = None): + """Launch an app. If activity is omitted, monkey-launch it.""" + if activity: + self._shell(f"am start -n {package}/{activity}") + else: + self._shell(f"monkey -p {package} -c android.intent.category.LAUNCHER 1") + + # ── helpers ─────────────────────────────────────────────── + + def _shell(self, cmd: str) -> str: + """Run 'adb -s shell '. Returns stdout decoded.""" + if not self.serial: + raise AdbError("Not connected — call connect() first") + return self._raw(f"-s {self.serial} shell {cmd}") + + def _exec_out(self, cmd: str) -> bytes: + """Run 'adb -s exec-out '. Returns raw bytes (for images).""" + if not self.serial: + raise AdbError("Not connected — call connect() first") + full = ["adb", "-s", self.serial, "exec-out"] + cmd.split() + logger.debug("ADB exec-out: %s", " ".join(full)) + r = subprocess.run(full, capture_output=True, timeout=30) + if r.returncode != 0: + raise AdbError(f"ADB exec-out failed: {r.stderr.decode(errors='replace')}") + return r.stdout + + def _raw(self, cmd: str) -> str: + """Run 'adb '. Returns decoded stdout.""" + full = ["adb"] + cmd.split() + logger.debug("ADB: %s", " ".join(full)) + r = subprocess.run(full, capture_output=True, text=True, timeout=30) + if r.returncode != 0: + raise AdbError(f"ADB error: {r.stderr.strip()}") + return r.stdout diff --git a/core/engine.py b/core/engine.py new file mode 100644 index 0000000..b5a8f89 --- /dev/null +++ b/core/engine.py @@ -0,0 +1,185 @@ +""" +Engine — the main loop that ties everything together: + screenshot → detect state → lookup playbook → run actions. +""" +import logging +import os +import time +import yaml + +from .adb import AdbClient +from .vision import Vision +from .action_loader import load_actions + +logger = logging.getLogger("sos.engine") + + +class Engine: + """ + Runs the bot loop. Each iteration: + 1. Capture screenshot (in-memory) + 2. Evaluate each playbook step (top-to-bottom) + 3. Execute the first matching step's action sequence + 4. Sleep the configured delay + """ + + def __init__(self, playbook_path: str, actions_dir: str = None): + self.playbook_path = playbook_path + self.actions_dir = actions_dir + self.adb = AdbClient() + self.vision = Vision(self.adb) + self.registry = None + self.running = False + self._step_count = 0 + + # ── lifecycle ──────────────────────────────────────────── + + def start(self): + """Load config, connect ADB, load actions, enter main loop.""" + import os + # Resolve paths + pb_path = os.path.abspath(self.playbook_path) + project_root = os.path.dirname(os.path.dirname(pb_path)) + self._templates_dir = os.path.join(project_root, "templates") + self._project_root = project_root + + # 1. Load playbook + with open(pb_path) as f: + config = yaml.safe_load(f) + self.config = config + logger.info("Loaded playbook: %s", self.playbook_path) + + # 2. Connect ADB + host = config.get("adb_host", "127.0.0.1") + port = config.get("adb_port", 5555) + self.adb = AdbClient(host=host, port=port) + self.adb.connect() + self.vision = Vision(self.adb) + + if not self.adb.is_connected(): + raise RuntimeError(f"ADB not connected to {host}:{port}") + + logger.info("ADB connected: %s", self.adb.serial) + + # 3. Load actions + actions_dir = self.actions_dir or os.path.join(project_root, "actions") + self.registry = load_actions(actions_dir) + + # 4. Main loop + self.running = True + self._loop() + + def stop(self): + """Signal the main loop to stop.""" + self.running = False + logger.info("Stop signalled — finishing current iteration") + + # ── main loop ──────────────────────────────────────────── + + def _loop(self): + delay = self.config.get("loop_delay", 2.0) + steps = self.config.get("steps", []) + + logger.info("Bot loop started (%d steps, delay=%.1fs)", len(steps), delay) + + while self.running: + self._step_count += 1 + try: + # 1. Capture + self.vision.capture() + + # 2. Evaluate steps in order + matched = False + for step in steps: + if self._evaluate_step(step): + logger.info( + "Step #%d — %s — matched → executing", + self._step_count, + step.get("name", "?"), + ) + self._execute_actions(step.get("then", [])) + matched = True + break # only first match per iteration + + if not matched: + logger.debug("Step #%d — no step matched", self._step_count) + + except KeyboardInterrupt: + self.running = False + break + except Exception as e: + logger.error("Loop error on step #%d: %s", self._step_count, e) + import traceback + traceback.print_exc() + + if self.running: + time.sleep(delay) + + logger.info("Bot loop ended (%d iterations)", self._step_count) + + # ── step evaluation ────────────────────────────────────── + + def _evaluate_step(self, step: dict) -> bool: + """ + Check if a playbook step's condition matches the current screen. + Conditions (checked in order): + - if_template: template matching + - if_pixel: pixel color check + - if_multiple: all templates must match (AND) + Returns True if the step's trigger condition is met. + """ + # Template match + if "if_template" in step: + tpl = step["if_template"] + conf = step.get("confidence", 0.8) + region = step.get("region") # optional [x1,y1,x2,y2] + tpl_path = os.path.join(self._templates_dir, tpl) + return self.vision.find_template(tpl_path, conf, region) is not None + + # Multiple templates (AND) + if "if_multiple" in step: + for tpl in step["if_multiple"]: + tpl_path = os.path.join(self._templates_dir, tpl) + conf = step.get("confidence", 0.8) + if self.vision.find_template(tpl_path, conf) is None: + return False + return True + + # Pixel color match + if "if_pixel" in step: + px = step["if_pixel"] + tol = px.get("tolerance", 10) + color = tuple(px["color"]) + return self.vision.color_match(px["x"], px["y"], color, tol) + + # Counter-based trigger (every N iterations) + if "if_counter" in step: + n = step["if_counter"] + return self._step_count > 0 and self._step_count % n == 0 + + # Default: always match (fallback / idle step) + return step.get("always", False) + + # ── action execution ───────────────────────────────────── + + def _execute_actions(self, actions: list): + """Run a sequence of actions.""" + ctx = { + "adb": self.adb, + "vision": self.vision, + "engine": self, + "screenshot": self.vision.current, + "log": logger, + "_templates_dir": self._templates_dir, + } + for item in actions: + name = item["action"] + params = item.get("params", {}) + try: + self.registry.run(name, ctx, params) + except KeyError as e: + logger.error("Action error: %s", e) + break + except Exception as e: + logger.error("Action '%s' failed: %s", name, e) + raise diff --git a/core/vision.py b/core/vision.py new file mode 100644 index 0000000..0aa7786 --- /dev/null +++ b/core/vision.py @@ -0,0 +1,180 @@ +""" +Vision — in-memory screenshot capture + OpenCV template matching. +No files are written to disk for screenshots; everything lives in numpy arrays. +""" +import logging +import numpy as np +import cv2 + +logger = logging.getLogger("sos.vision") + + +class Vision: + """Captures screenshots and provides template-matching utilities.""" + + def __init__(self, adb_client): + self.adb = adb_client + self._screenshot = None # current frame (BGR numpy array) + self._screenshot_gray = None # grayscale version (cached) + self._size = None # (width, height) + + # ── capture ────────────────────────────────────────────── + + def capture(self) -> np.ndarray: + """ + Take a fresh screenshot from the device. + Returns BGR numpy array. Never touches disk. + """ + png_bytes = self.adb.screencap() + # Decode PNG from memory buffer + arr = np.frombuffer(png_bytes, dtype=np.uint8) + img = cv2.imdecode(arr, cv2.IMREAD_COLOR) + if img is None: + raise RuntimeError("Failed to decode screencap — corrupt PNG data?") + self._screenshot = img + self._screenshot_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + self._size = (img.shape[1], img.shape[0]) + return img + + @property + def current(self) -> np.ndarray: + """Return the most recent screenshot (call capture() first).""" + if self._screenshot is None: + raise RuntimeError("No screenshot — call capture() first") + return self._screenshot + + @property + def size(self): + """(width, height) of the device screen.""" + if self._size is None: + self.capture() + return self._size + + # ── template matching ──────────────────────────────────── + + def find_template( + self, + template_path: str, + confidence: float = 0.8, + region: tuple = None, + ) -> tuple | None: + """ + Locate a template image on the current screenshot. + + Args: + template_path: Path to the reference PNG. + confidence: Minimum match ratio (0-1). Higher = stricter. + region: (x1, y1, x2, y2) subregion to search, or None = full. + + Returns: + (center_x, center_y) of the best match, or None. + """ + template_rgb = cv2.imread(template_path, cv2.IMREAD_COLOR) + if template_rgb is None: + logger.warning("Cannot load template: %s", template_path) + return None + + template_gray = cv2.cvtColor(template_rgb, cv2.COLOR_BGR2GRAY) + tw, th = template_gray.shape[1], template_gray.shape[0] + + # Use the cached grayscale screenshot + search_img = self._screenshot_gray + + # Crop to region if specified + if region: + x1, y1, x2, y2 = region + search_img = search_img[y1:y2, x1:x2] + + if search_img.shape[0] < th or search_img.shape[1] < tw: + logger.debug("Template %s bigger than search region", template_path) + return None + + result = cv2.matchTemplate(search_img, template_gray, cv2.TM_CCOEFF_NORMED) + _, max_val, _, max_loc = cv2.minMaxLoc(result) + + if max_val < confidence: + logger.debug( + "Template %s: best=%.3f < threshold=%.3f", + template_path, max_val, confidence, + ) + return None + + cx = max_loc[0] + tw // 2 + cy = max_loc[1] + th // 2 + if region: + cx += region[0] + cy += region[1] + + logger.info( + "Found %s at (%.3f conf) → click (%d, %d)", + template_path, max_val, cx, cy, + ) + return (cx, cy) + + def find_multiple( + self, + template_path: str, + confidence: float = 0.8, + max_results: int = 5, + region: tuple = None, + ) -> list[tuple]: + """ + Find ALL matches of a template (non-max-suppressed). + + Returns list of (cx, cy) centers sorted by confidence descending. + """ + template_rgb = cv2.imread(template_path, cv2.IMREAD_COLOR) + if template_rgb is None: + return [] + template_gray = cv2.cvtColor(template_rgb, cv2.COLOR_BGR2GRAY) + tw, th = template_gray.shape[1], template_gray.shape[0] + + search_img = self._screenshot_gray + if region: + x1, y1, x2, y2 = region + search_img = search_img[y1:y2, x1:x2] + + result = cv2.matchTemplate(search_img, template_gray, cv2.TM_CCOEFF_NORMED) + locations = np.where(result >= confidence) + points = [] + for pt in zip(*locations[::-1]): # (x, y) pairs + cx = pt[0] + tw // 2 + cy = pt[1] + th // 2 + if region: + cx += region[0] + cy += region[1] + points.append((cx, cy)) + + # Non-max suppression: group nearby detections + if not points: + return [] + points = np.array(points) + indices = cv2.dnn.NMSBoxes( + [(*p, tw, th) for p in points], + [1.0] * len(points), + 0.5, + 0.3, + ) + if len(indices) == 0: + return [] + result_pts = [tuple(points[i]) for i in indices.flatten()] + return result_pts[:max_results] + + # ── pixel checking ─────────────────────────────────────── + + def pixel_color(self, x: int, y: int) -> tuple: + """Get the BGR color of a single pixel on the current screenshot.""" + return tuple(self.current[y, x].tolist()) + + def color_match(self, x: int, y: int, target_bgr: tuple, tolerance: int = 10) -> bool: + """Check if pixel at (x,y) is within tolerance of target_bgr.""" + actual = self.pixel_color(x, y) + return all(abs(actual[c] - target_bgr[c]) <= tolerance for c in range(3)) + + # ── debug ──────────────────────────────────────────────── + + def save_debug_screenshot(self, path: str): + """Save current screenshot to disk for debugging (only when requested).""" + if self._screenshot is not None: + cv2.imwrite(path, self._screenshot) + logger.info("Debug screenshot saved: %s", path) diff --git a/debug_screenshot.py b/debug_screenshot.py new file mode 100644 index 0000000..dec264e --- /dev/null +++ b/debug_screenshot.py @@ -0,0 +1,77 @@ +""" +Debug screenshot tool — capture a clean screenshot from the emulator +and save it so you can crop reference templates from it. + +Usage: + source .venv/bin/activate + python debug_screenshot.py # saves sos_screenshot_.png + python debug_screenshot.py --output my_screen.png + python debug_screenshot.py --port 5556 +""" +import argparse +import logging +import os +import sys +from datetime import datetime + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from core.adb import AdbClient +from core.vision import Vision + + +def main(): + parser = argparse.ArgumentParser( + description="Capture a debug screenshot from the emulator" + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=5555) + parser.add_argument("--output", "-o", help="Output PNG path") + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + + level = logging.DEBUG if args.verbose else logging.INFO + logging.basicConfig(level=level, format="%(message)s") + log = logging.getLogger("sos") + + if not args.output: + ts = datetime.now().strftime("%Y%m%d_%H%M%S") + args.output = f"sos_screenshot_{ts}.png" + + print(f"[*] Connecting ADB to {args.host}:{args.port}...") + adb = AdbClient(args.host, args.port) + adb.connect() + + if not adb.is_connected(): + print("[!] ADB not connected. Is BlueStacks running with ADB enabled?") + sys.exit(1) + + print(f"[*] ADB connected: {adb.serial}") + vision = Vision(adb) + + print("[*] Capturing screenshot (in-memory)...") + vision.capture() + + w, h = vision.size + print(f"[*] Device resolution: {w}x{h}") + + print(f"[*] Saving to {args.output}...") + vision.save_debug_screenshot(args.output) + + # Also print pixel info utility + print(f"\n[*] Saved: {args.output} ({os.path.getsize(args.output)} bytes)") + print() + print("NEXT STEPS:") + print(f" 1. Open {args.output} in an image editor") + print(f" 2. Crop the button/icon you want to click") + print(f" 3. Save the crop to templates/.png") + print(f" 4. Add a step in playbook/default.yaml referencing it") + print() + print("PIXEL CHECK: read a pixel color from the saved image:") + print(f" python -c \"from PIL import Image; im=Image.open('{args.output}'); print(im.getpixel((100,200)))\"") + print() + + adb.disconnect() + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 0000000..2c2bfb3 --- /dev/null +++ b/main.py @@ -0,0 +1,87 @@ +""" +State of Survival Bot — Main Entry Point +Usage: + python main.py # default playbook & port + python main.py --playbook my_play.yaml + python main.py --port 5556 + python main.py --host 192.168.1.100 +""" +import argparse +import logging +import signal +import sys +import os + +# Ensure the project root is on sys.path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from core.engine import Engine + + +def setup_logging(verbose: bool = False): + level = logging.DEBUG if verbose else logging.INFO + fmt = "%(asctime)s [%(levelname)-5s] %(message)s" + datefmt = "%H:%M:%S" + logging.basicConfig(level=level, format=fmt, datefmt=datefmt) + # Keep OpenCV chatter quiet + logging.getLogger("cv2").setLevel(logging.WARNING) + + +def main(): + parser = argparse.ArgumentParser( + description="State of Survival Bot — ADB + OpenCV automation", + ) + parser.add_argument( + "--playbook", + default=os.path.join(os.path.dirname(__file__), "playbook", "default.yaml"), + help="Path to playbook YAML (default: playbook/default.yaml)", + ) + parser.add_argument( + "--host", + default="127.0.0.1", + help="ADB host (default: 127.0.0.1)", + ) + parser.add_argument( + "--port", + type=int, + default=5555, + help="ADB port (default: 5555)", + ) + parser.add_argument( + "-v", "--verbose", + action="store_true", + help="Verbose debug logging", + ) + args = parser.parse_args() + + setup_logging(args.verbose) + logger = logging.getLogger("sos") + + engine = Engine(args.playbook) + engine.playbook_path = args.playbook + engine.adb_host = args.host + engine.adb_port = args.port + + # Handle Ctrl+C gracefully + def sigint_handler(sig, frame): + logger.info("Received SIGINT — shutting down...") + engine.stop() + signal.signal(signal.SIGINT, sigint_handler) + + try: + engine.start() + except KeyboardInterrupt: + engine.stop() + except Exception as e: + logger.error("Fatal: %s", e) + sys.exit(1) + finally: + try: + engine.adb.disconnect() + except Exception: + pass + logger.info("Bot shut down") + + +if __name__ == "__main__": + main() diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..3b2208a --- /dev/null +++ b/plan.md @@ -0,0 +1,713 @@ +# State of Survival Bot — Implementation Plan + +## Overview + +Automated bot for State of Survival running in BlueStacks. Uses ADB + OpenCV template matching. All screenshots stay in memory. Behavior driven by YAML playbook. New actions added as standalone `.py` files in `actions/`. + +--- + +## Phase 1 — Foundation (DONE) + +### 1.1 Framework + +| Module | File | Status | +|--------|------|--------| +| ADB client | `core/adb.py` | ✅ Done | +| Vision (template matching) | `core/vision.py` | ✅ Done | +| Action loader (plugin system) | `core/action_loader.py` | ✅ Done | +| Engine (main loop) | `core/engine.py` | ✅ Done | +| Entry point | `main.py` | ✅ Done | +| Debug screenshot tool | `debug_screenshot.py` | ✅ Done | +| Default playbook | `playbook/default.yaml` | ✅ Done | +| 12 built-in actions | `actions/*.py` | ✅ Done | + +### 1.2 BlueStacks Setup (manual, one-time) + +1. BlueStacks → Settings → Advanced → **Android Debug Bridge: ON** +2. BlueStacks → Settings → Display → **Resolution: 1600×900, DPI: 240** +3. Install State of Survival, log in +4. `adb connect 127.0.0.1:5555` +5. `adb devices` — confirm "device" state + +--- + +## Phase 2 — Screen Detection System + +### 2.1 Main Screen Detection + +**Goal:** Reliably know when we're on the settlement main screen vs. a sub-menu, dialog, or loading screen. + +**Strategy:** Template matching on 2-3 unique static UI elements. + +| Template | What to crop | Suggested file | +|----------|-------------|----------------| +| Bottom nav bar icon (settlement) | The settlement/globe icon at bottom-left | `main_screen_marker.png` | +| Resource bar (any corner) | A section of the resource bar at the top | `resource_bar.png` | +| Left menu button | The menu button (~22, 478) | `left_menu_btn.png` | + +**Playbook step:** +```yaml +- name: "Verify main screen" + if_multiple: ["main_screen_marker.png", "resource_bar.png"] + confidence: 0.8 + then: + - action: log + params: { message: "On main screen" } +``` + +**Negative check (not on main screen):** +```yaml +- name: "Not on main screen — force restart" + if_not_template: "main_screen_marker.png" # <-- needs engine support + then: + - action: app + params: { action: "restart", package: "com.kingsgroup.sos", wait: 15 } +``` + +### 2.2 Popup / Ad Detection + +**Goal:** Detect and dismiss all overlay dialogs, ads, and offers. + +| Template | Location | Confidence | +|----------|----------|------------| +| `close_ad.png` | X button in top-right of ad banners | 0.75 | +| `dialog_close.png` | X button for dialogs | 0.75 | +| `offer_close.png` | Small X on special offers | 0.7 | +| `back_btn.png` | Back arrow top-left | 0.75 | + +**Region constraint** for top-right X buttons: `[1440, 0, 1600, 300]` + +**Strategy:** Check for known close buttons every cycle, before any other action. Multiple close attempts in sequence for layered dialogs (some offers have 2-3 popups stacked). + +```yaml +steps: + - name: "Close ads (layer 1)" + if_template: "close_ad.png" + region: [1440, 0, 1600, 300] + then: + - action: find_and_click + params: { template: "close_ad.png", region: [1440, 0, 1600, 300] } + - action: wait + params: { seconds: 2 } + + - name: "Close ads (layer 2)" + if_template: "dialog_close.png" + region: [1480, 0, 1600, 100] + then: + - action: find_and_click + params: { template: "dialog_close.png", region: [1480, 0, 1600, 100] } +``` + +--- + +## Phase 3 — Resource Collection + +### 3.1 Collect Finished Gathers + +**Goal:** Detect completed gathering nodes on the world map and collect resources. + +| Template | What to crop | Notes | +|----------|-------------|-------| +| `collect_btn.png` | "Collect" button that appears on finished nodes | Crop tight, green text on dark bg | +| `gather_complete_icon.png` | The checkmark/ribbon on gathering buildings | Visible from settlement | + +**Flow:** +1. Scan settlement for gather-complete markers +2. Click marker → opens node dialog +3. Click "Collect" button +4. Close result dialog + +**Playbook:** +```yaml +- name: "Collect resources" + if_template: "gather_complete_icon.png" + confidence: 0.7 + then: + - action: find_and_click + params: { template: "gather_complete_icon.png" } + - action: wait + params: { seconds: 2 } + - action: find_and_click + params: { template: "collect_btn.png" } + - action: wait + params: { seconds: 2 } + - action: tap + params: { x: 1553, y: 32 } # close +``` + +--- + +## Phase 4 — Military Actions + +### 4.1 Train Troops + +**Goal:** Keep training queues active for all troop types. + +**Flow:** +1. Open left menu (tap 22, 478) +2. Tap troop type 1 (550, 182) +3. Tap building center (806, 432) twice +4. Check if training icon exists (932, 645) +5. If training icon exists → check if button is "Train" or "Already Training" +6. If Train → click it, else close +7. Repeat for troop types 2 (550, 261) and 3 (550, 348) + +| Template | What to crop | Notes | +|----------|-------------|-------| +| `troops_building.png` | The barracks/training building on screen | For find-and-click | +| `train_btn.png` | The "Train" button text/icon | Green button | +| `training_in_progress.png` | The indicator that training is active | Grayed out or timer | +| `troop_type_1.png` | Right-menu troop icon line 1 | Region-crop | +| `troop_type_2.png` | Right-menu troop icon line 2 | | +| `troop_type_3.png` | Right-menu troop icon line 3 | | + +**Strategy choice:** We have two approaches: + +**A) Coordinate-based** (simpler, brittle if UI shifts): +```yaml +- name: "Train troops type 1" + if_counter: 5 + then: + - action: tap + params: { x: 22, y: 478 } # left menu + - action: wait + params: { seconds: 1 } + - action: tap + params: { x: 550, y: 182 } # troop type 1 + - action: wait + params: { seconds: 2 } + - action: tap + params: { x: 806, y: 432 } # building center + - action: wait + params: { seconds: 1 } + - action: find_and_click + params: { template: "train_btn.png", confidence: 0.75 } + - action: tap + params: { x: 1553, y: 32 } # close +``` + +**B) Template-based** (more robust): +```yaml +- name: "Find barracks and train" + if_template: "troops_building.png" + confidence: 0.7 + then: + - action: find_and_click + params: { template: "troops_building.png" } + - action: wait + params: { seconds: 2 } + - action: find_and_click + params: { template: "train_btn.png", confidence: 0.75 } + - action: wait + params: { seconds: 1 } + - action: tap + params: { x: 1553, y: 32 } +``` + +**Recommendation:** Start with approach A, add templates as fallbacks. + +### 4.2 Heal Troops + +**Goal:** Heal wounded troops from the hospital. + +| Template | What to crop | +|----------|-------------| +| `heal_btn.png` | Hospital/heal button | +| `heal_all.png` | "Heal All" button | +| `hospital_icon.png` | Hospital building icon on settlement map | + +**Flow:** +1. Find hospital on screen → tap +2. Wait for dialog +3. Click "Heal All" +4. Close + +### 4.3 Attack Warning → Shield + +**Goal:** When an attack is incoming, activate a shield. + +| Template | What to crop | +|----------|-------------| +| `attack_banner.png` | Red attack warning banner at top of screen | +| `shield_2h.png` | 2-hour shield icon/button | +| `shield_8h.png` | 8-hour shield icon/button | +| `confirm_shield.png` | Confirm / activate shield button | + +**Flow (from original code):** +1. Check pixel (1525, 468) for red color (195, 69, 69) — attack warning indicator +2. Tap (1401, 850) — open rally response panel +3. Tap (90, 600) — switch to world chat to see the warning +4. Search region (182-861, 80-263) for green shield buttons +5. Tap shield → tap confirm (1338, 825) +6. Close dialogs + +**Template version:** +```yaml +- name: "Shield on attack" + if_template: "attack_banner.png" + confidence: 0.7 + then: + - action: find_and_click + params: { template: "shield_2h.png", confidence: 0.7 } + - action: wait + params: { seconds: 1 } + - action: find_and_click + params: { template: "confirm_shield.png", confidence: 0.7 } + - action: wait + params: { seconds: 2 } + - action: tap + params: { x: 1553, y: 32 } + - action: log + params: { message: "SHIELD ACTIVATED" } +``` + +### 4.4 Join Rally Groups + +**Goal:** Join alliance rallies against infected/marauders when available. + +**Flow (from original code):** +1. Look for group invitation banner (amber color ~174,114,40) at top-right (1540-1584, 305-348) +2. Click to open group screen +3. Search for available rally in region (1227-1527, 102-882) — olive color ~95,120,46 +4. Check stamina — if ended, close and return +5. Check march limit — if limited, close and return +6. Check empty troops — if no troops selected, tap clear (773, 789) then add 1 (1289, 279) +7. March (1269, 801) +8. Handle "not enough troops" message if it appears + +| Template | What to crop | +|----------|-------------| +| `rally_invite_banner.png` | Amber banner at top | +| `march_btn.png` | "March" button | +| `stamina_empty.png` | Indicator that stamina is depleted | +| `rally_available.png` | A rally you can join in the list | + +--- + +## Phase 5 — Progression Actions + +### 5.1 Upgrade Buildings + +**Goal:** Upgrade buildings when upgrade arrows are visible. + +**Flow:** +1. Tap building slot 1 (63, 240) — first column, first row +2. Wait for building dialog +3. Scan buildings area (576-1059, 431-768) for green upgrade arrow (4 shades of green) +4. If upgrade arrow found → tap it → opens upgrade dialog +5. Check if "Upgrade" button is active (not gray ~85,85,85) +6. If active → tap Upgrade +7. If gray → resources missing → close +8. Close dialog +9. Repeat for building slot 2 (63, 350) + +| Template | What to crop | +|----------|-------------| +| `upgrade_arrow_green.png` | Green arrow pointing up on a building | +| `upgrade_btn.png` | "Upgrade" button in upgrade dialog (active, not gray) | +| `upgrade_gray.png` | Grayed out upgrade button (missing resources) | +| `building_slot_1.png` | First building position on left column | +| `building_slot_2.png` | Second building position on left column | + +### 5.2 Research + +**Goal:** Progress research trees (Fight and Economy). + +**Flow:** +1. Open left menu (22, 478) +2. Tap research tab (550, 493) +3. Open research screen (987, 605) +4. Check if research is already in progress (pixel 489, 844 == brown ~123,82,57) → if yes, close +5. Toggle between Fight branch (652, 835) and Economy branch (1437, 838) +6. Tap "Go" (1158, 823) +7. Confirm (831, 301) +8. Close + +| Template | What to crop | +|----------|-------------| +| `research_tab.png` | Research icon in left menu | +| `research_in_progress.png` | "In Progress" indicator | +| `research_go_btn.png` | "Go" / "Start Research" button | +| `research_fight_tab.png` | Fight research branch | +| `research_economy_tab.png` | Economy research branch | + +### 5.3 Intelligence Collection + +**Goal:** Collect completed intelligence mission rewards. + +**Flow:** +1. Open left menu (22, 478) +2. Check pixel (35, 568) for orange-brown color (~157, 94, 77) — indicates rewards available +3. If reward available → tap intelligence screen (551, 624) +4. Tap Collect (1115, 607) +5. Close (1188, 305) +6. If no reward → close left menu (668, 446) + +| Template | What to crop | +|----------|-------------| +| `intel_tab.png` | Intelligence tab in left menu | +| `intel_notification.png` | Orange dot/indicator on intel tab | +| `intel_collect_btn.png` | "Collect" button in intel screen | + +--- + +## Phase 6 — Alliance Actions + +### 6.1 Alliance Tech Donations + +**Goal:** Contribute to alliance technology research. + +**Flow:** +1. Open clan (1140, 848) +2. Tap tech tab (1482, 542) +3. Wait 3s for loading +4. Tap recommendations filter (1005, 42) +5. Scan region (257-1500, 174-804) for red recommendation indicators (~114,27,27) +6. If found → tap it 7× (opens the donation cart) +7. Click 11 rounds of donation slots (3 per round: 614, 675, 749 at x=1149) +8. Close cart (1333, 140) +9. Return (46, 33) +10. Send alliance help handshake (890, 730) → (848, 802) +11. Open gift box (1495, 745) +12. Collect Ali's Gift (1386, 347) → Big Gift (363, 411) +13. Close ads, collect all green checkmarks in gift panel + +| Template | What to crop | +|----------|-------------| +| `clan_btn.png` | Clan/alliance button on main screen | +| `tech_tab.png` | Technology tab | +| `donate_btn.png` | Donate button | +| `alliance_help_btn.png` | Handshake/help button | +| `gift_box_btn.png` | Gift box button | +| `red_dot_recommendation.png` | Red dot on recommended tech | + +### 6.2 Alliance Gift Collection + +**Goal:** Collect daily gifts from alliance members. + +**Flow:** +1. Open gift box → Ali's Gift → Big Gift +2. Close ad popups that appear +3. Collect all gifts (scan for green checkmark color ~112,145,54) +4. Collect the final big reward button + +--- + +## Phase 7 — World Map / Intel + +### 7.1 Intel Radar Missions + +**Goal:** Auto-run radar intel missions for rewards. + +**Flow:** +1. Open wilderness (77, 827) +2. Tap Search (71, 704) +3. Tap Intel tab (126, 645) +4. Go Search (1362, 822) +5. Click center of map (797, 365) +6. Check for available target (brown ~137,88,32) or raid button (olive ~100,124,42) +7. If target → check march count, level, send march +8. Wait for result (10s intervals): + - Win: yellow text (255,221,162) at (597, 202) → click center 4× + - Lose: olive text (76,101,35) at (1186, 637) → close +9. Also collect already-completed intel markers (red ~157,44,31) + +| Template | What to crop | +|----------|-------------| +| `wilderness_btn.png` | Wilderness toggle button bottom-left | +| `search_btn.png` | Search button on world map | +| `intel_tab_world.png` | Intel tab in search panel | +| `go_search_btn.png` | "Search" / "Go" button | +| `march_btn_intel.png` | March button in intel target dialog | +| `intel_win_text.png` | Victory text indicator | +| `intel_lose_text.png` | Defeat text indicator | +| `intel_collect_marker.png` | Red marker for completed intel | + +### 7.2 Hero Raid Stages + +**Goal:** Auto-run hero raid stages. + +**Flow (from original code, currently commented out):** +1. Open wilderness → settlement +2. Tap Build Hero (1139, 518) +3. Open Raid (1401, 507) +4. Open Stage (1112, 476) +5. Scan for red enemy markers (~243,51,68) up to 6 scans +6. If found → tap marker → Go Fight (1028, 801) +7. Wait for win/lose (same as intel) +8. Repeat until no markers found + +| Template | What to crop | +|----------|-------------| +| `hero_building.png` | Hero building icon | +| `raid_btn.png` | Raid button inside hero screen | +| `enemy_marker_red.png` | Red enemy icon on raid map | +| `fight_btn.png` | "Fight" / "Attack" button | +| `stage_select.png` | Stage selector | + +--- + +## Phase 8 — Utility Actions + +### 8.1 App Restart on Hang + +**Goal:** Force-restart the game when stuck on loading screen or frozen. + +**Strategy:** +- `if_counter: 180` (every ~9 minutes at 3s loop) — proactive restart +- Or after 3 consecutive "not main screen" detections + +**Implementation:** +```yaml +- name: "Periodic restart" + if_counter: 180 + then: + - action: app + params: { action: "restart", package: "com.kingsgroup.sos", wait: 15 } +``` + +### 8.2 Swipe / Navigate + +**Gesture helpers for custom scenarios:** + +| Action | Implementation | +|--------|---------------| +| Swipe up | `swipe: { x1: 250, y1: 700, x2: 250, y2: 600, duration_ms: 500 }` | +| Swipe left | `swipe: { x1: 767, y1: 449, x2: 255, y2: 449, duration_ms: 500 }` | +| Swipe right | `swipe: { x1: 255, y1: 449, x2: 767, y2: 449, duration_ms: 500 }` | +| Open left menu | `tap: { x: 22, y: 478 }` | +| Close dialog X | `tap: { x: 1553, y: 32 }` | +| Back button | `keyevent: { key: "back" }` | +| Toggle world/settlement | `tap: { x: 77, y: 827 }` | + +--- + +## Phase 9 — Advanced Features (Future) + +### 9.1 OCR for Reading Values + +**Add Tesseract or EasyOCR to read:** +- Resource counts (wood, food, metal, oil) +- March/troop counts +- Timers (how long until building/training finishes) +- VIP level, power score + +**New actions needed:** +- `actions/ocr_read.py` — reads text from a screen region → returns string +- `actions/ocr_click.py` — finds text via OCR and taps it + +### 9.2 State Machine / Decision Tree + +Replace flat playbook step evaluation with a proper FSM: +- States: `MAIN_SCREEN`, `TRAINING`, `HEALING`, `UPGRADING`, `INTEL`, `OFF_SCREEN`, `POPUP` +- Transitions based on template detection +- Timeouts per state (don't stay in INTEL for more than 60s) +- Counter for consecutive "off screen" detections → trigger restart + +### 9.3 Multi-Instance Support + +- Loop over multiple ADB ports (5555, 5557, 5559…) +- Each instance runs its own engine thread +- Shared template directory +- Per-instance playbook or config override + +### 9.4 Statistics & Logging + +- Log kills, resources gained, shield activations +- Export stats to CSV +- Discord/Telegram webhook notifications on events (attack, shield, level-up) +- Screenshot on error for debugging + +--- + +## Action Checklist — Master Table + +| # | Action | Phase | Priority | Detection Method | Templates Needed | +|---|--------|-------|----------|-----------------|------------------| +| 1 | Main screen detection | 2 | 🔴 Critical | Template × 2 | `main_screen_marker.png`, `resource_bar.png` | +| 2 | Close popup ads | 2 | 🔴 Critical | Template (region) | `close_ad.png`, `dialog_close.png` | +| 3 | Attack → shield | 4 | 🔴 Critical | Template | `attack_banner.png`, `shield_2h.png`, `confirm_shield.png` | +| 4 | Restart on hang | 8 | 🟡 High | Counter | None | +| 5 | Collect resources | 3 | 🟡 High | Template | `gather_complete_icon.png`, `collect_btn.png` | +| 6 | Train troops | 4 | 🟡 High | Coord + Template | `train_btn.png`, `troops_building.png` | +| 7 | Heal troops | 4 | 🟡 High | Template | `heal_btn.png`, `heal_all.png` | +| 8 | Upgrade buildings | 5 | 🟡 High | Pixel + Template | `upgrade_arrow_green.png`, `upgrade_btn.png` | +| 9 | Research | 5 | 🟡 High | Coord + Pixel | `research_in_progress.png`, `research_go_btn.png` | +| 10 | Intelligence collect | 5 | 🟡 High | Coord + Pixel | `intel_notification.png`, `intel_collect_btn.png` | +| 11 | Alliance tech donate | 6 | 🟢 Medium | Coord + Pixel | `clan_btn.png`, `tech_tab.png`, `red_dot_recommendation.png` | +| 12 | Alliance gifts | 6 | 🟢 Medium | Coord | None (coordinate-based) | +| 13 | Alliance help handshake | 6 | 🟢 Medium | Coord + Template | `alliance_help_btn.png` | +| 14 | Join rallies | 4.4 | 🟢 Medium | Template | `rally_invite_banner.png`, `march_btn.png` | +| 15 | Intel radar missions | 7 | 🔵 Low | Coord + Pixel | `intel_tab_world.png`, `march_btn_intel.png` | +| 16 | Hero raid stages | 7.2 | 🔵 Low | Pixel | `enemy_marker_red.png`, `fight_btn.png` | +| 17 | OCR reading | 9 | 🔵 Low | Tesseract | None (text ↔ image) | +| 18 | FSM state machine | 9 | 🔵 Low | Engine upgrade | None | +| 19 | Multi-instance | 9 | ⚪ Later | Threading | None | +| 20 | Webhook notifications | 9 | ⚪ Later | HTTP | None | + +--- + +## Recommended Build Order + +``` +Phase 2: Screens → Popups → Main detection [Day 1 — you're here] + ↓ +Phase 3: Collect resources [Day 1] + ↓ +Phase 4: Shield → Train → Heal → Join rallies [Day 2] + ↓ +Phase 5: Upgrade → Research → Intel [Day 2-3] + ↓ +Phase 6: Alliance actions [Day 3] + ↓ +Phase 7: Intel missions → Hero raids [Day 4] + ↓ +Phase 9: OCR → FSM → Multi-instance → Webhooks [Ongoing] +``` + +--- + +## How to Create Templates (Step by Step) + +### Method 1: Using debug_screenshot.py + +```bash +cd ~/sos_bot +source .venv/bin/activate + +# 1. Capture current screen +python debug_screenshot.py -o screen.png + +# 2. Open screen.png in GIMP/Krita +# 3. Crop around a button (e.g., the Train button) +# 4. Save as templates/train_btn.png +``` + +### Method 2: In-game pixel coordinate finder + +If you're on the main screen and want to verify coordinates match the bot's expectations, use: + +```bash +source .venv/bin/activate +python3 -c " +from PIL import Image +im = Image.open('screen.png') +# Get color at any (x, y) +print(im.getpixel((1533, 615))) # main screen check pixel +print(im.getpixel((1525, 468))) # attack banner check +" +``` + +### Template Design Rules + +| Rule | Why | +|------|-----| +| Crop **tightly** around the target | Less background = fewer false positives | +| Include the **whole button** (border to border) | Partial buttons match poorly | +| Use **PNG** format (lossless) | JPG artifacts reduce match confidence | +| Start with `confidence: 0.8` | Lower to 0.7 if not found, never below 0.6 | +| Use `region:` to limit search area | Speeds up matching, reduces false positives | +| Keep templates under 50×50 px where possible | Smaller templates = faster matching | +| For **dynamic** elements (timers, counts) | Use OCR instead (Phase 9) | +| For **text** buttons | Template works if font is consistent | +| For **icons** (shield, heal, x button) | Template works perfectly — icons are static | + +--- + +## Playbook File Organization + +As we add features, keep the playbook clean: + +### Option A: Single file with sections (recommended for now) + +``` +playbook/default.yaml + ↓ + Steps organized by function: + 1. Ad/Popup closing (high priority) + 2. Attack/Shield (urgent) + 3. Resource collection + 4. Training/Healing + 5. Building/Research + 6. Alliance + 7. Intel/Raid + 8. Idle/Restart +``` + +### Option B: Multiple playbook files (future) + +``` +playbook/ +├── default.yaml # main config + includes +├── core.yaml # detection, popups, restart +├── military.yaml # train, heal, shield, rallies +├── progression.yaml # build, research, intel +├── alliance.yaml # donate, gifts, help +└── world.yaml # search, intel missions, raids +``` + +**Requires** `!include` YAML extension or pre-processor. + +--- + +## Custom Actions to Build + +We'll need these new actions beyond the 12 already built: + +| Action | File | Purpose | +|--------|------|---------| +| `pixel_check` | `actions/pixel_check.py` | Check pixel color, return True/False for branching | +| `loop_until` | `actions/loop_until.py` | Repeat sub-actions until a condition is met | +| `click_while` | `actions/click_while.py` | Keep clicking a position until screen changes | +| `scroll_find` | `actions/scroll_find.py` | Scroll a list until a template appears | +| `random_delay` | `actions/random_delay.py` | Human-like variable delay (anti-detection) | +| `screenshot_debug` | `actions/screenshot_debug.py` | Save screenshot on demand (for debugging) | +| `notify` | `actions/notify.py` | Discord/Telegram webhook notification | + +### Pixel Check Action (needed for branching logic) + +```python +# actions/pixel_check.py +class Action: + name = "pixel_check" + + def run(self, ctx, params): + x = params["x"] + y = params["y"] + target = tuple(params["color"]) + tolerance = params.get("tolerance", 10) + match = ctx["vision"].color_match(x, y, target, tolerance) + ctx["_last_pixel_match"] = match + ctx["log"].info("pixel_check(%d,%d) = %s", x, y, match) + return match +``` + +--- + +## Engine Improvements Needed + +| # | Feature | File | What to change | +|---|---------|------|----------------| +| 1 | `if_not_template` condition | `core/engine.py` | Negation of template match | +| 2 | Step cooldown / rate limit | `core/engine.py` | Don't run same step more than once per N seconds | +| 3 | Action result → context | `core/engine.py` | Pass action return values into `ctx` for subsequent actions | +| 4 | Step timeout guard | `core/engine.py` | If a step's actions take > 60s, abort and re-screenshot | +| 5 | Consecutive failure counter | `core/engine.py` | After 3 failures, restart app | + +--- + +## Next Steps — Right Now + +1. Start BlueStacks with SoS running +2. `adb connect 127.0.0.1:5555` +3. `python debug_screenshot.py` — capture your first screen +4. Open the screenshot in GIMP, crop these first templates: + - The close/X button in top-right corner → `close_ad.png` + - The settlement/globe icon bottom-left → `main_screen_marker.png` + - The attack banner (if visible) → `attack_banner.png` +5. Put them in `templates/` +6. Uncomment/simplify `playbook/default.yaml` to test `close_ad` + `main_screen_marker` +7. Run `python main.py` — verify detection works +8. Capture more screenshots in each game state → crop more templates +9. Add steps to playbook one at a time +10. After each new step, run for 5+ minutes to verify it doesn't get stuck diff --git a/playbook/default.yaml b/playbook/default.yaml new file mode 100644 index 0000000..660501d --- /dev/null +++ b/playbook/default.yaml @@ -0,0 +1,181 @@ +# ============================================================================= +# State of Survival Bot — Playbook +# ============================================================================= +# Steps are evaluated TOP-to-BOTTOM each loop. The FIRST matching step runs. +# Put more specific conditions first, generic fallback last. +# +# HOW TO ADD NEW BEHAVIOR: +# 1. Take a screenshot: python debug_screenshot.py +# 2. Crop the target button/icon from the saved image +# 3. Save the crop as templates/your_template.png +# 4. Add a step below referencing it via "if_template" +# ============================================================================= + +adb_host: "127.0.0.1" +adb_port: 5555 +loop_delay: 3 # seconds between iterations + +steps: + + # ────────────────────────────────────────────────────────── + # CLOSE POPUPS + # ────────────────────────────────────────────────────────── + - name: "Close popup ad (X button top-right)" + if_template: "close_ad.png" + confidence: 0.75 + region: [1440, 0, 1600, 300] + then: + - action: find_and_click + params: + template: "close_ad.png" + confidence: 0.75 + region: [1440, 0, 1600, 300] + - action: wait + params: { seconds: 2 } + + - name: "Close generic dialog (X)" + if_template: "dialog_close.png" + confidence: 0.75 + region: [1480, 0, 1600, 100] + then: + - action: find_and_click + params: + template: "dialog_close.png" + confidence: 0.75 + region: [1480, 0, 1600, 100] + - action: wait + params: { seconds: 1 } + + - name: "Press BACK to dismiss overlays" + if_template: "back_btn.png" + confidence: 0.7 + region: [0, 0, 100, 150] + then: + - action: keyevent + params: { key: "back" } + - action: log + params: { message: "Back pressed (dismiss overlay)" } + + # ────────────────────────────────────────────────────────── + # COLLECT / GATHER + # ────────────────────────────────────────────────────────── + - name: "Collect from finished gathering" + if_template: "collect_btn.png" + confidence: 0.75 + then: + - action: find_and_click + params: { template: "collect_btn.png" } + - action: wait + params: { seconds: 2 } + - action: tap + params: { x: 1300, y: 700 } + - action: wait + params: { seconds: 2 } + - action: tap + params: { x: 1553, y: 32 } + - action: log + params: { message: "Resources collected" } + + # ────────────────────────────────────────────────────────── + # HEAL TROOPS + # ────────────────────────────────────────────────────────── + - name: "Heal wounded troops" + if_template: "heal_btn.png" + confidence: 0.75 + then: + - action: find_and_click + params: { template: "heal_btn.png" } + - action: wait + params: { seconds: 2 } + - action: find_and_click + params: { template: "heal_all.png", confidence: 0.7 } + - action: wait + params: { seconds: 2 } + - action: tap + params: { x: 1553, y: 32 } + - action: log + params: { message: "Troops healed" } + + # ────────────────────────────────────────────────────────── + # TRAIN TROOPS + # ────────────────────────────────────────────────────────── + - name: "Train troops" + if_template: "train_btn.png" + confidence: 0.75 + then: + - action: find_and_click + params: { template: "train_btn.png" } + - action: wait + params: { seconds: 3 } + - action: tap + params: { x: 800, y: 750 } + - action: wait + params: { seconds: 1 } + - action: tap + params: { x: 1553, y: 32 } + - action: log + params: { message: "Training queued" } + + # ────────────────────────────────────────────────────────── + # UPGRADE BUILDINGS + # ────────────────────────────────────────────────────────── + - name: "Find & upgrade building (slot 1)" + if_template: "upgrade_arrow.png" + confidence: 0.7 + region: [50, 200, 200, 600] + then: + - action: tap + params: { x: 63, y: 240 } + - action: wait + params: { seconds: 2 } + - action: find_and_click + params: { template: "upgrade_btn.png", confidence: 0.7 } + - action: wait + params: { seconds: 1 } + - action: tap + params: { x: 1553, y: 32 } + - action: log + params: { message: "Building upgrade attempted" } + + # ────────────────────────────────────────────────────────── + # ALLIANCE HELP + # ────────────────────────────────────────────────────────── + - name: "Send alliance help" + if_template: "alliance_help_btn.png" + confidence: 0.75 + then: + - action: find_and_click + params: { template: "alliance_help_btn.png" } + - action: wait + params: { seconds: 1 } + - action: click_seq + params: + coords: [[1149, 614], [1149, 675], [1149, 749]] + delay: 0.3 + - action: tap + params: { x: 46, y: 33 } # back to settlement + - action: log + params: { message: "Alliance help sent" } + + # ────────────────────────────────────────────────────────── + # RESTART: every 180th iteration (≈9min at 3s delay) + # ────────────────────────────────────────────────────────── + - name: "Periodic restart (every 180 cycles)" + if_counter: 180 + then: + - action: app + params: + action: "restart" + package: "com.kingsgroup.sos" + wait: 15 + - action: log + params: { message: "Periodic restart" } + + # ────────────────────────────────────────────────────────── + # IDLE FALLBACK + # ────────────────────────────────────────────────────────── + - name: "Idle — nothing matched" + always: true + then: + - action: wait + params: { seconds: 5 } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a764e26 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +opencv-python-headless>=4.9 +numpy>=1.24 +pyyaml>=6.0 +pillow>=10.0 diff --git a/templates/README.txt b/templates/README.txt new file mode 100644 index 0000000..2f61194 --- /dev/null +++ b/templates/README.txt @@ -0,0 +1,27 @@ +# Place your reference template PNGs here. +# +# Naming convention: +# main_screen_marker.png — a unique UI element that identifies the main screen +# collect_btn.png — collect / gather button +# heal_btn.png — heal troops button +# train_btn.png — train troops button +# upgrade_available.png — green upgrade indicator over a building +# upgrade_btn.png — the "Upgrade" button in the building dialog +# attack_banner.png — attack warning banner +# shield_2h.png — 2-hour shield icon +# confirm_shield.png — confirm shield deployment +# close_ad.png — close button for ads/popups +# dialog_close.png — X close button for dialogs +# +# How to create templates: +# 1. Let the bot take a debug screenshot: +# python -c " +# from core.adb import AdbClient; from core.vision import Vision +# a = AdbClient('127.0.0.1', 5555); a.connect() +# v = Vision(a); v.capture() +# v.save_debug_screenshot('debug_screen.png') +# " +# 2. Crop the relevant button/icon with an image editor +# 3. Save as PNG in this directory +# 4. Reference the filename in playbook/default.yaml +#