Merge branch 'master' into u/cuza/fixes-for-vscode-test-parser

.activate.sh

@@ -1 +1 @@
-.tox/py37-linux/bin/activate
+.tox/py38-linux/bin/activate

.github/workflows/ci.yml

@@ -17,7 +17,10 @@ jobs:
       fail-fast: false
       matrix:
         toxenv:
-          - py38-linux,docs,mypy,tests
+          - py38-linux
+          - docs
+          - mypy
+          - tests
           - general_itests
     env:
       DOCKER_REGISTRY: ""

.gitignore

@@ -43,12 +43,14 @@ paasta_itests/fake_etc_paasta/marathon.json
 yelp_package/gopath
 .mypy_cache/
 .pytest_cache/
+.hypothesis/
 debian/.debhelper
 example_cluster/paasta/docker_registry.json
 general_itests/fake_etc_paasta/clusters.json
 pip-wheel-metadata
 debian/debhelper-build-stamp
 unique-run
+.vault-token
 
 # Coverage artifacts
 .coverage

.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# RTD defaults as of 2023-11-08
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: requirements-docs.txt

Makefile

@@ -23,8 +23,10 @@ endif
 
 ifeq ($(PAASTA_ENV),YELP)
 	export DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/
+	export DOCKER_OPT_ARGS ?=
 else
-	export DOCKER_REGISTRY ?= ""
+	export DOCKER_REGISTRY ?= docker.io/
+	export DOCKER_OPT_ARGS ?= --user `id -u`:`id -g`
 	export INDEX_URL_BUILD_ARG ?= PIP_INDEX_URL
 endif
 
@@ -50,7 +52,7 @@ test-not-yelpy: .paasta/bin/activate
 	.paasta/bin/tox -e tests
 
 quick-test: .tox/py38-linux
-	TZ=UTC .tox/py38-linux/bin/py.test --last-failed -x -- tests
+	TZ=UTC .tox/py38-linux/bin/py.test --failed-first -x --disable-warnings -- tests
 
 .tox/py38-linux: .paasta/bin/activate
 	.paasta/bin/tox
@@ -115,8 +117,8 @@ k8s_clean: .paasta/bin/activate
 #   in paasta repo: java -jar ~/openapi-generator/modules/openapi-generator-cli/target/openapi-generator-cli.jar
 openapi-codegen:
 	rm -rf paasta_tools/paastaapi
-	docker run --rm -i --user `id -u`:`id -g` -v `pwd`:/src -w /src \
-		yelp/openapi-generator-cli:20201026 \
+	docker run --rm -i ${DOCKER_OPT_ARGS} -v `pwd`:/src -w /src \
+		${DOCKER_REGISTRY}yelp/openapi-generator-cli:20201026 \
 		generate \
 		-i paasta_tools/api/api_docs/oapi.yaml \
 		-g python-experimental \
@@ -127,8 +129,8 @@ openapi-codegen:
 	rm -rf temp-openapi-client
 
 swagger-validate:
-	docker run --rm -i --user `id -u`:`id -g` -v `pwd`:/src -w /src \
-		yelp/openapi-generator-cli:20201026 \
+	docker run --rm -i ${DOCKER_OPT_ARGS} -v `pwd`:/src -w /src \
+		${DOCKER_REGISTRY}yelp/openapi-generator-cli:20201026 \
 		validate \
 		-i paasta_tools/api/api_docs/swagger.json
 
@@ -158,13 +160,33 @@ setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service
 	export PAASTA_TEST_CLUSTER=kind-${USER}-k8s-test;\
 	.tox/py38-linux/bin/python -m paasta_tools.list_kubernetes_service_instances -d ./soa_config_playground --shuffle --group-lines 1 | xargs --no-run-if-empty .tox/py38-linux/bin/python -m paasta_tools.setup_kubernetes_job -d ./soa_config_playground -c kind-${USER}-k8s-test
 
+.PHONY: cleanup-kubernetes-jobs
+cleanup-kubernetes-jobs:
+	export KUBECONFIG=./k8s_itests/kubeconfig;\
+	export PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_playground/;\
+	export PAASTA_TEST_CLUSTER=kind-${USER}-k8s-test;\
+	.tox/py38-linux/bin/python -m paasta_tools.cleanup_kubernetes_jobs -d ./soa_config_playground -c kind-${USER}-k8s-test --force
+
 .PHONY: paasta-secrets-sync
 paasta-secrets-sync: setup-kubernetes-job .vault-token
 	export KUBECONFIG=./k8s_itests/kubeconfig;\
 	export PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_playground/;\
 	export PAASTA_TEST_CLUSTER=kind-${USER}-k8s-test;\
 	{ .tox/py38-linux/bin/python -m paasta_tools.list_kubernetes_service_instances -d ./soa_config_playground ; echo -n \ _shared; } | cut -f1 -d"." | uniq | shuf | xargs .tox/py38-linux/bin/python -m paasta_tools.kubernetes.bin.paasta_secrets_sync -v -d ./soa_config_playground -t ./.vault-token
 
+define ANNOUNCE_CRONS_BODY
+The following PaaSTA cron jobs will run on an infinite loop using the PaaSTA Playground k8s cluster:
+- setup-kubernetes-job
+- cleanup-kubernetes-job
+- paasta-secrets-sync
+- generate_deployments_for_service
+endef
+export ANNOUNCE_CRONS_BODY
+.PHONY: paasta-crons
+make paasta-cronjobs:
+	@echo "$$ANNOUNCE_CRONS_BODY"
+	while true; do make paasta-secrets-sync && make cleanup-kubernetes-jobs; sleep 5; done
+
 .vault-token:
 	export VAULT_ADDR=https://vault-devc.yelpcorp.com:8200 ;\
 	export VAULT_SKIP_VERIFY=true ;\

README.md

@@ -5,45 +5,61 @@
 ![PaaSTA Logo](http://engineeringblog.yelp.com/images/previews/paasta_preview.png)
 
 PaaSTA is a highly-available, distributed system for building, deploying, and
-running services using containers and Apache Mesos!
+running services using containers and Kubernetes.
+
+PaaSTA has been running production services at Yelp since 2016. It was
+originally designed to run on top of Apache Mesos but has subsequently been
+updated to use Kubernetes. Over time the features and functionality that
+PaaSTA provides have increased but the principal design remains the same.
+
+PaaSTA aims to take a declarative description of the services that teams need
+to run and then ensures that those services are deployed safely, efficiently,
+and in a manner that is easy for the teams to maintain. Rather than managing
+Kubernetes YAML files, PaaSTA provides a simplified schema to describe your service
+and in addition to configuring Kubernetes it can also configure other infrastructure
+tools to provide monitoring, logging, cost management etc.
 
 Want to know more about the opinions behind what makes PaaSTA special? Check
 out the [PaaSTA Principles](http://paasta.readthedocs.io/en/latest/about/paasta_principles.html).
 
-*Note*: PaaSTA has been running in production at Yelp for years,
-and has a number of "Yelpisms" still lingering in the codebase. We have made
-efforts to excise them, but there are bound to be lingering issues. Please help us
-by opening an [issue](https://github.com/Yelp/paasta/issues/new) or
-better yet a [pull request](https://github.com/Yelp/paasta/pulls).
-
 ## Components
 
 *Note*: PaaSTA is an opinionated platform that uses a few un-opinionated
 tools. It requires a non-trivial amount of infrastructure to be in place
 before it works completely:
 
  * [Docker](http://www.docker.com/) for code delivery and containment
- * [Mesos](http://mesos.apache.org/) / [Kubernetes](https://kubernetes.io/) for code execution and scheduling (runs Docker containers)
- * [Marathon](https://mesosphere.github.io/marathon/) for managing long-running services
+ * [Kubernetes](https://kubernetes.io/) for code execution and scheduling (runs Docker containers)
  * [Tron](https://tron.readthedocs.io/en/latest/) for running things on a timer (nightly batches)
- * [SmartStack](http://nerds.airbnb.com/smartstack-service-discovery-cloud/) / [Envoy](https://www.envoyproxy.io/) for service registration and discovery
- * [Sensu](https://sensuapp.org/) for monitoring/alerting
+ * [SmartStack](http://nerds.airbnb.com/smartstack-service-discovery-cloud/) and [Envoy](https://www.envoyproxy.io/) for service registration and discovery
+ * [Sensu](https://sensu.io/) for monitoring/alerting
  * [Jenkins](https://jenkins-ci.org/) (optionally) for continuous deployment
+ * [Prometheus](https://prometheus.io/) and [HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) for autoscaling services
+
+One advantage to having a PaaS composed of components like these is you
+get to reuse them for other purposes. For example, at Yelp Sensu is not just for
+PaaSTA, it can be used to monitor all sorts of things. We also use Kubernetes to run
+other more complex workloads like [Jolt](https://dcos.io/events/2017/jolt-distributed-fault-tolerant-tests-at-scale-on-mesos/) and [Cassandra](https://engineeringblog.yelp.com/2020/11/orchestrating-cassandra-on-kubernetes-with-operators.html). Our service mesh, which
+is a heavily customised version of SmartStack and Envoy, allows many systems at Yelp
+to communicate with PaaSTA services and each other.
+
+On the other hand, requiring lots of components, means lots of infrastructure to
+setup before PaaSTA can work effectively! Realistacally, running PaaSTA outside of Yelp
+would not be sensible, because in addition to the integrations mentioned above we also
+have strong opinions encoded in other tooling that you would need to replicate. Nevertheless,
+we code PaaSTA in the open because we think it is useful to share our approach and hope that
+the code can at least help others understand or solve similar problems.
 
-The main advantage to having a PaaS composed of components like these is you
-get to reuse them for other purposes. For example at Yelp Sensu is not just for
-PaaSTA, it can be used to monitor all sorts of things. Also Mesos can be
-re-used for things like custom frameworks. For example at Yelp we use the Mesos
-infrastructure to run our large-scale testing framework:
-[Jolt](https://dcos.io/events/2017/jolt-distributed-fault-tolerant-tests-at-scale-on-mesos/).
-SmartStack is used at Yelp for service discovery for Non-PaaSTA things as well,
-like databases, legacy apps, and Puppet-defined apps. Most PaaS's do not
-allow for this type of component re-use.
-
-On the other hand, requiring lots of components means lots of infrastructure to
-setup before PaaSTA is fully baked. If you are looking for a project that
-doesn't require external components, we encourage you to look at the doc
-[comparing PaaSTA to other tools](https://github.com/Yelp/paasta/blob/master/comparison.md).
+## Integrations and Features
+
+In addition to the direct integrations above PaaSTA also relies on other components
+to provide PaaSTA users with other features and to manage compute capacity at Yelp.
+
+* We use [Karpenter](https://karpenter.sh/) to autoscale pools of EC2 instances to run PaaSTA. Formerly we used our own autoscaler [Clusterman](https://engineeringblog.yelp.com/2019/11/open-source-clusterman.html)
+* We bake AMIs using [Packer](https://www.packer.io/)
+* We collect logs from services and send them via [Monk](https://engineeringblog.yelp.com/2020/01/streams-and-monk-how-yelp-approaches-kafka-in-2020.html) to [Kafka](https://kafka.apache.org/)
+* We use [StatefulSets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) to run a few stateful PaaSTA services
+* We autotune the resources needed by each service by monitoring usage (similar to [VPA](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler))
 
 ## Design Goals
 
@@ -54,16 +70,11 @@ doesn't require external components, we encourage you to look at the doc
  * No single points of failure
  * Pleasant interface
 
-PaaSTA is an opinionated platform, and it is not designed to interoperate with
-every possible backend service out there.
-
-Think of it as an example of how we have integrated these technologies together
-to build a cohesive PaaS. It is not a turn-key PaaS solution.
-
 ## Getting Started
 
 See the [getting started](http://paasta.readthedocs.io/en/latest/installation/getting_started.html)
-documentation for how to deploy PaaSTA.
+documentation for how to deploy PaaSTA. This reference is intended to help understand how PaaSTA
+works but we don't advise that you use PaaSTA in production.
 
 ## Debugging PaaSTA (in VS Code)
 

contrib/python-k8s-client.diff

@@ -0,0 +1,11 @@
+--- a/debian/paasta/opt/venvs/paasta/lib/python3.8/site-packages/kubernetes/client/api_client.py
++++ b/debian/paasta/opt/venvs/paasta/lib/python3.8/site-packages/kubernetes/client/api_client.py
+@@ -629,7 +629,7 @@
+                                                    'get_real_child_model'):
+             return data
+
+-        kwargs = {}
++        kwargs = {"local_vars_configuration": self.configuration}
+         if (data is not None and
+                 klass.openapi_types is not None and
+                 isinstance(data, (list, dict))):