From ba202eca05c96b559ffa26a3d61b52c8e2ae9d97 Mon Sep 17 00:00:00 2001 From: SporeStack Date: Thu, 13 Apr 2023 00:01:33 +0000 Subject: [PATCH] v10.0.0: Cleanups/refactor Added integration-test.sh --- .gitignore | 1 + CHANGELOG.md | 7 ++ integration-test.sh | 59 ++++++++++++ pyproject.toml | 2 + src/sporestack/__init__.py | 2 +- src/sporestack/api.py | 4 +- src/sporestack/api_client.py | 179 ++++++++++++++++------------------- src/sporestack/cli.py | 50 +++++----- src/sporestack/exceptions.py | 6 ++ tox.ini | 4 +- 10 files changed, 188 insertions(+), 126 deletions(-) create mode 100755 integration-test.sh diff --git a/.gitignore b/.gitignore index 2f01335..8404c40 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ __pycache__ .pytest_cache .coverage .tox +dummydotsporestackfolder diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f06ef0..d8572fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [10.0.0 - 2023-04-12] + +## Changed + +- No more `retry` options in `api_client`. Use try/except for `SporeStackServerError`, instead, to retry on 500s. +- Exception messages may be improved. + ## [9.1.1 - 2023-04-12] ### Changed diff --git a/integration-test.sh b/integration-test.sh new file mode 100755 index 0000000..7219024 --- /dev/null +++ b/integration-test.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# These are pretty hacky and need to be cleaned up, but serve a purpose. + +# Set REAL_TESTING_TOKEN for more tests. + +set -ex + +export SPORESTACK_DIR=$(pwd)/dummydotsporestackfolder + +rm -r $SPORESTACK_DIR +mkdir $SPORESTACK_DIR + +sporestack version +sporestack version | grep '[0-9]\.[0-9]\.[0-9]' + +sporestack api-endpoint +sporestack api-endpoint | grep api.sporestack.com + +sporestack token list +sporestack token list 2>&1 | wc -l | grep '2$' + +sporestack token import importediminvalid --key "imaninvalidkey" +sporestack token list | grep importediminvalid +sporestack token list | grep imaninvalidkey +sporestack server launch --no-quote --token neverbeencreated --operating-system debian-11 --days 1 2>&1 | grep 'does not exist' + +# Online tests start here. + +sporestack token create --dollars 50 --currency fakecurrency ihaveafakecurrency 2>&1 | grep 'value is not a valid' +sporestack server launch --no-quote --token importediminvalid --operating-system debian-11 --days 1 2>&1 | grep 'ensure this value has at least 32' + +sporestack server flavors | grep vcpu +sporestack server operating-systems | grep debian-11 + +if [ -z "$REAL_TESTING_TOKEN" ]; then + echo "REAL_TESTING_TOKEN not set, not finishing tests." + echo Success + exit 0 +else + echo "REAL_TESTING_TOKEN is set, will continue testing." +fi + +sporestack token import realtestingtoken --key "$REAL_TESTING_TOKEN" +sporestack token balance realtestingtoken | grep -F '$' +sporestack token messages realtestingtoken +sporestack token servers realtestingtoken + +sporestack server launch --no-quote --token realtestingtoken --operating-system debian-11 --days 1 --hostname sporestackpythonintegrationtestdelme +sporestack server topup --token realtestingtoken --hostname sporestackpythonintegrationtestdelme --days 1 +sporestack server info --token realtestingtoken --hostname sporestackpythonintegrationtestdelme +sporestack server json --token realtestingtoken --hostname sporestackpythonintegrationtestdelme +sporestack server start --token realtestingtoken --hostname sporestackpythonintegrationtestdelme +sporestack server stop --token realtestingtoken --hostname sporestackpythonintegrationtestdelme +sporestack server rebuild --token realtestingtoken --hostname sporestackpythonintegrationtestdelme +sporestack server delete --token realtestingtoken --hostname sporestackpythonintegrationtestdelme +sporestack server forget --token realtestingtoken --hostname sporestackpythonintegrationtestdelme + +echo Success diff --git a/pyproject.toml b/pyproject.toml index d09247d..f1fe66f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,8 @@ ignore = [ unfixable = [ "F401", # Don't try to automatically remove unused imports + "RUF100", # Unused noqa + "F841", # Unused variable ] target-version = "py37" diff --git a/src/sporestack/__init__.py b/src/sporestack/__init__.py index 8b40939..28b7b82 100644 --- a/src/sporestack/__init__.py +++ b/src/sporestack/__init__.py @@ -2,4 +2,4 @@ __all__ = ["api", "api_client", "exceptions"] -__version__ = "9.1.1" +__version__ = "10.0.0" diff --git a/src/sporestack/api.py b/src/sporestack/api.py index 9a93212..2d75ac9 100644 --- a/src/sporestack/api.py +++ b/src/sporestack/api.py @@ -7,7 +7,7 @@ SporeStack API request/response models from datetime import datetime from enum import Enum -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union from pydantic import BaseModel, Field @@ -85,7 +85,7 @@ class ServerTopup: class Request(BaseModel): days: int - token: str + token: Union[str, None] = None class ServerInfo: diff --git a/src/sporestack/api_client.py b/src/sporestack/api_client.py index 17862e2..c49ca34 100644 --- a/src/sporestack/api_client.py +++ b/src/sporestack/api_client.py @@ -1,8 +1,7 @@ import logging import os from dataclasses import dataclass -from time import sleep -from typing import Any, Dict, List, Optional +from typing import List, Optional import httpx from pydantic import parse_obj_as @@ -59,6 +58,40 @@ def _is_onion_url(url: str) -> bool: return False +def _get_response_error_text(response: httpx.Response) -> str: + """Get a response's error text. Assumes the response is actually an error.""" + if ( + "content-type" in response.headers + and response.headers["content-type"] == "application/json" + ): + error = response.json() + if "detail" in error: + if isinstance(error["detail"], str): + return error["detail"] + else: + return str(error["detail"]) + + return response.text + + +def _handle_response(response: httpx.Response) -> None: + status_code_first_digit = response.status_code // 100 + if status_code_first_digit == 2: + return + + error_response_text = _get_response_error_text(response) + if response.status_code == 429: + raise exceptions.SporeStackTooManyRequestsError(error_response_text) + elif status_code_first_digit == 4: + raise exceptions.SporeStackServerError(error_response_text) + elif status_code_first_digit == 5: + # User should probably retry. + raise exceptions.SporeStackServerError(error_response_text) + else: + # How did we get here? + raise exceptions.SporeStackServerError(error_response_text) + + @dataclass class APIClient: api_endpoint: str = API_ENDPOINT @@ -70,67 +103,6 @@ class APIClient: proxy = _get_tor_proxy() self._httpx_client = httpx.Client(headers=headers, proxies=proxy) - def _api_request( - self, - url: str, - empty_post: bool = False, - json_params: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - retry: bool = False, - ) -> Any: - try: - if empty_post is True: - request = self._httpx_client.post(url, timeout=POST_TIMEOUT) - elif json_params is None: - request = self._httpx_client.get(url, timeout=GET_TIMEOUT) - else: - request = self._httpx_client.post( - url, - json=json_params, - timeout=POST_TIMEOUT, - ) - except Exception as e: - if retry is True: - log.warning(f"Got an error, but retrying: {e}") - sleep(5) - # Try again. - return self._api_request( - url, - empty_post=empty_post, - json_params=json_params, - retry=retry, - ) - else: - raise - - status_code_first_digit = request.status_code // 100 - if status_code_first_digit == 2: - try: - return request.json() - except Exception: - return request.content - elif status_code_first_digit == 4: - log.debug("HTTP status code: {request.status_code}") - raise exceptions.SporeStackUserError(request.content.decode("utf-8")) - elif status_code_first_digit == 5: - if retry is True: - log.warning(request.content.decode("utf-8")) - log.warning("Got a 500, retrying in 5 seconds...") - sleep(5) - # Try again if we get a 500 - return self._api_request( - url, - empty_post=empty_post, - json_params=json_params, - retry=retry, - ) - else: - raise exceptions.SporeStackServerError(str(request.content)) - else: - # Not sure why we'd get this. - request.raise_for_status() - raise Exception("Stuff broke strangely. Please contact SporeStack support.") - def server_launch( self, machine_id: str, @@ -155,31 +127,30 @@ class APIClient: autorenew=autorenew, ) url = self.api_endpoint + api.ServerLaunch.url.format(machine_id=machine_id) - self._api_request(url=url, json_params=request.dict()) + response = self._httpx_client.post(url=url, json=request.dict()) + _handle_response(response) def server_topup( self, machine_id: str, days: int, - token: str, + token: str | None = None, ) -> None: """Topup a server.""" request = api.ServerTopup.Request(days=days, token=token) url = self.api_endpoint + api.ServerTopup.url.format(machine_id=machine_id) - self._api_request(url=url, json_params=request.dict()) + response = self._httpx_client.post(url=url, json=request.dict()) + _handle_response(response) def server_quote(self, days: int, flavor: str) -> api.ServerQuote.Response: """Get a quote for how much a server will cost.""" url = self.api_endpoint + api.ServerQuote.url response = self._httpx_client.get( - url=url, + url, params={"days": days, "flavor": flavor}, ) - if response.status_code == 422: - raise exceptions.SporeStackUserError(response.json()["detail"]) - - response.raise_for_status() + _handle_response(response) return api.ServerQuote.Response.parse_obj(response.json()) def autorenew_enable(self, machine_id: str) -> None: @@ -189,7 +160,8 @@ class APIClient: url = self.api_endpoint + api.ServerEnableAutorenew.url.format( machine_id=machine_id ) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def autorenew_disable(self, machine_id: str) -> None: """ @@ -198,35 +170,40 @@ class APIClient: url = self.api_endpoint + api.ServerDisableAutorenew.url.format( machine_id=machine_id ) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def server_start(self, machine_id: str) -> None: """ Power on the server. """ url = self.api_endpoint + api.ServerStart.url.format(machine_id=machine_id) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def server_stop(self, machine_id: str) -> None: """ Power off the server. """ url = self.api_endpoint + api.ServerStop.url.format(machine_id=machine_id) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def server_delete(self, machine_id: str) -> None: """ Delete the server. """ url = self.api_endpoint + api.ServerDelete.url.format(machine_id=machine_id) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def server_forget(self, machine_id: str) -> None: """ Forget about a destroyed/deleted server. """ url = self.api_endpoint + api.ServerForget.url.format(machine_id=machine_id) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def server_rebuild(self, machine_id: str) -> None: """ @@ -235,15 +212,17 @@ class APIClient: Deletes all of the data on the server! """ url = self.api_endpoint + api.ServerRebuild.url.format(machine_id=machine_id) - self._api_request(url, empty_post=True) + response = self._httpx_client.post(url) + _handle_response(response) def server_info(self, machine_id: str) -> api.ServerInfo.Response: """ Returns info about the server. """ url = self.api_endpoint + api.ServerInfo.url.format(machine_id=machine_id) - response = self._api_request(url) - response_object = api.ServerInfo.Response.parse_obj(response) + response = self._httpx_client.get(url) + _handle_response(response) + response_object = api.ServerInfo.Response.parse_obj(response.json()) return response_object def servers_launched_from_token( @@ -253,22 +232,27 @@ class APIClient: Returns info of servers launched from a given token. """ url = self.api_endpoint + api.ServersLaunchedFromToken.url.format(token=token) - response = self._api_request(url) - response_object = api.ServersLaunchedFromToken.Response.parse_obj(response) + response = self._httpx_client.get(url) + _handle_response(response) + response_object = api.ServersLaunchedFromToken.Response.parse_obj( + response.json() + ) return response_object def flavors(self) -> api.Flavors.Response: """Returns available flavors (server sizes).""" url = self.api_endpoint + api.Flavors.url - response = self._api_request(url) - response_object = api.Flavors.Response.parse_obj(response) + response = self._httpx_client.get(url) + _handle_response(response) + response_object = api.Flavors.Response.parse_obj(response.json()) return response_object def operating_systems(self) -> api.OperatingSystems.Response: """Returns available operating systems.""" url = self.api_endpoint + api.OperatingSystems.url - response = self._api_request(url) - response_object = api.OperatingSystems.Response.parse_obj(response) + response = self._httpx_client.get(url) + _handle_response(response) + response_object = api.OperatingSystems.Response.parse_obj(response.json()) return response_object def token_add( @@ -276,30 +260,28 @@ class APIClient: token: str, dollars: int, currency: str, - retry: bool = False, ) -> api.TokenAdd.Response: """Add balance (money) to a token.""" - request = api.TokenAdd.Request(dollars=dollars, currency=currency) url = self.api_endpoint + api.TokenAdd.url.format(token=token) - response = self._api_request(url=url, json_params=request.dict(), retry=retry) - response_object = api.TokenAdd.Response.parse_obj(response) + request = api.TokenAdd.Request(dollars=dollars, currency=currency) + response = self._httpx_client.post(url, json=request.dict()) + _handle_response(response) + response_object = api.TokenAdd.Response.parse_obj(response.json()) return response_object def token_balance(self, token: str) -> api.TokenBalance.Response: """Return a token's balance.""" url = self.api_endpoint + api.TokenBalance.url.format(token=token) - response = self._api_request(url=url) - response_object = api.TokenBalance.Response.parse_obj(response) + response = self._httpx_client.get(url) + _handle_response(response) + response_object = api.TokenBalance.Response.parse_obj(response.json()) return response_object def token_get_messages(self, token: str) -> List[api.TokenMessage]: """Get messages for/from the token.""" url = self.api_endpoint + f"/token/{token}/messages" - log.debug(f"Token send message URL: {url}") response = self._httpx_client.get(url=url) - if response.status_code == 422: - raise exceptions.SporeStackUserError(response.json()["detail"]) - response.raise_for_status() + _handle_response(response) return parse_obj_as(List[api.TokenMessage], response.json()) @@ -307,7 +289,4 @@ class APIClient: """Send a message to SporeStack support.""" url = self.api_endpoint + f"/token/{token}/messages" response = self._httpx_client.post(url=url, json={"message": message}) - if response.status_code == 422: - raise exceptions.SporeStackUserError(response.json()["detail"]) - - response.raise_for_status() + _handle_response(response) diff --git a/src/sporestack/cli.py b/src/sporestack/cli.py index 8c5ffd1..325670c 100644 --- a/src/sporestack/cli.py +++ b/src/sporestack/cli.py @@ -30,7 +30,7 @@ _home = os.getenv("HOME", None) assert _home is not None, "Unable to detect $HOME environment variable?" HOME = Path(_home) -SPORESTACK_DIR = HOME / ".sporestack" +SPORESTACK_DIR = Path(os.getenv("SPORESTACK_DIR", HOME / ".sporestack")) # Try to protect files in ~/.sporestack os.umask(0o0077) @@ -211,7 +211,11 @@ def server_info_path() -> Path: servers_dir = SPORESTACK_DIR / "servers" # Migrate existing server.json files into servers subdirectory - if SPORESTACK_DIR.exists() and not servers_dir.exists(): + if ( + SPORESTACK_DIR.exists() + and not servers_dir.exists() + and len(list(SPORESTACK_DIR.glob("*.json"))) > 0 + ): typer.echo( f"Migrating server profiles found in {SPORESTACK_DIR} to {servers_dir}.", err=True, @@ -627,6 +631,7 @@ def token_create( raise typer.Exit(1) from .api_client import APIClient + from .exceptions import SporeStackServerError api_client = APIClient(api_endpoint=get_api_endpoint()) @@ -634,7 +639,6 @@ def token_create( token=_token, dollars=dollars, currency=currency, - retry=True, ) uri = response.payment.uri @@ -650,12 +654,15 @@ def token_create( # FIXME: Wait two hours in a smarter way. # Waiting for payment to set in. time.sleep(10) - response = api_client.token_add( - token=_token, - dollars=dollars, - currency=currency, - retry=True, - ) + try: + response = api_client.token_add( + token=_token, + dollars=dollars, + currency=currency, + ) + except SporeStackServerError: + typer.echo("Received 500 HTTP status, will try again.", err=True) + continue if response.payment.paid is True: typer.echo(f"{token} has been enabled with ${dollars}.") typer.echo(f"{token}'s key is {_token}.") @@ -688,6 +695,7 @@ def token_topup( token = load_token(token) from .api_client import APIClient + from .exceptions import SporeStackServerError api_client = APIClient(api_endpoint=get_api_endpoint()) @@ -695,7 +703,6 @@ def token_topup( token, dollars, currency=currency, - retry=True, ) uri = response.payment.uri @@ -709,12 +716,15 @@ def token_topup( typer.echo(WAITING_PAYMENT_TO_PROCESS, err=True) tries = tries - 1 # FIXME: Wait two hours in a smarter way. - response = api_client.token_add( - token, - dollars, - currency=currency, - retry=True, - ) + try: + response = api_client.token_add( + token=token, + dollars=dollars, + currency=currency, + ) + except SporeStackServerError: + typer.echo("Received 500 HTTP status, will try again.", err=True) + continue # Waiting for payment to set in. time.sleep(10) if response.payment.paid is True: @@ -789,9 +799,7 @@ def messages(token: str = typer.Argument(DEFAULT_TOKEN)) -> None: def send_message( token: str = typer.Argument(DEFAULT_TOKEN), message: str = typer.Option(...) ) -> None: - """ - Send a support message. - """ + """Send a support message.""" token = load_token(token) from .api_client import APIClient @@ -805,9 +813,7 @@ def send_message( @cli.command() def version() -> None: - """ - Returns the installed version. - """ + """Returns the installed version.""" from . import __version__ typer.echo(__version__) diff --git a/src/sporestack/exceptions.py b/src/sporestack/exceptions.py index 80a7d15..9483fbb 100644 --- a/src/sporestack/exceptions.py +++ b/src/sporestack/exceptions.py @@ -8,6 +8,12 @@ class SporeStackUserError(SporeStackError): pass +class SporeStackTooManyRequestsError(SporeStackError): + """HTTP 429, retry again later""" + + pass + + class SporeStackServerError(SporeStackError): """HTTP 5XX""" diff --git a/tox.ini b/tox.ini index 60976ae..5e10753 100644 --- a/tox.ini +++ b/tox.ini @@ -12,4 +12,6 @@ deps = pytest-socket~=0.6.0 pytest-cov~=4.0 pytest-mock~=3.6 -commands = pytest --cov=sporestack --cov-fail-under=39 --cov-report=term --durations=3 +commands = + pytest --cov=sporestack --cov-fail-under=39 --cov-report=term --durations=3 + sporestack api-endpoint