feat: support nacos storage

.github/workflows/build-and-test.yaml

@@ -134,7 +134,41 @@ jobs:
 
     - name: "Run Higress E2E Conformance Tests"
       run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test
-
+
+  higress-conformance-test-nacos:
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Setup Golang Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-go
+
+      - name: Setup Submodule Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            .git/modules
+          key: ${{ runner.os }}-submodules-cache-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-submodules-cache
+
+      - run: git stash # restore patch
+
+      - name: "Run Higress E2E Conformance Nacos Tests"
+        run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test-nacos
+
   publish:
     runs-on: ubuntu-latest
     needs: [higress-conformance-test,gateway-conformance-test]

Makefile.core.mk

@@ -184,6 +184,10 @@ install-dev: pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'
 install-dev-wasmplugin: build-wasmplugins pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'  --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false'
+install-dev-nacos: pre-install
+	kubectl apply -f tools/hack/conf/nacos.yaml
+	tools/hack/gen-secret-configmap.sh
+	helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"'
 
 uninstall:
 	helm uninstall higress -n higress-system
@@ -241,6 +245,15 @@ higress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-bui
 .PHONY: higress-wasmplugin-test
 higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-higress-e2e-test-wasmplugin delete-cluster
 
+# higress-conformance-test-nacos runs ingress nacos tests.
+.PHONY: higress-conformance-test-nacos
+higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-nacos delete-cluster
+
+.PHONY: port-forward-nacos
+port-forward-nacos:
+	kubectl wait --timeout=10m -n higress-system deploy/nacos --for=condition=Available
+	kubectl port-forward -n higress-system svc/nacos-service 8848:8848 9848:9848 &
+
 # create-cluster creates a kube cluster with kind.
 .PHONY: create-cluster
 create-cluster: $(tools/kind)
@@ -259,13 +272,13 @@ delete-cluster: $(tools/kind) ## Delete kind cluster.
 kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG.
 	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)
 	tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
-	tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
+	tools/hack/docker-pull-image.sh docker.io/swsk33/nacos-standalone 2.2.3
 	tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0
 	tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
 	tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest
 	tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2
 	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
-	tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
+	tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.3
 	tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0
 	tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2
 	tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
@@ -289,3 +302,12 @@ run-higress-e2e-test-wasmplugin:
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
 	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true
+
+.PHONY: run-higress-e2e-test-nacos
+run-higress-e2e-test-nacos:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true

docker/Dockerfile.higress

@@ -13,7 +13,11 @@ FROM ${HUB}/base:${BASE_VERSION}
 ARG TARGETARCH
 
 COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress
+COPY wait-for-apiserver.sh /usr/local/bin/wait-for-apiserver.sh
+COPY run-higress.sh /usr/local/bin/run-higress.sh
+
+RUN chmod +x /usr/local/bin/*.sh
 
 USER 1337:1337
 
-ENTRYPOINT ["/usr/local/bin/higress"]
+ENTRYPOINT ["/usr/local/bin/run-higress.sh"]

docker/docker.mk

@@ -14,12 +14,16 @@
 
 docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress: $(OUT_LINUX)/higress
+docker.higress: docker/wait-for-apiserver.sh
+docker.higress: docker/run-higress.sh
 docker.higress: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_RULE)
 
 docker.higress-buildx: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress-buildx: $(AMD64_OUT_LINUX)/higress
 docker.higress-buildx: $(ARM64_OUT_LINUX)/higress
+docker.higress: docker/wait-for-apiserver.sh
+docker.higress: docker/run-higress.sh
 docker.higress-buildx: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_BUILDX_RULE)
 

docker/run-higress.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#  Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       http:www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+args=("$@")
+
+for arg in "${args[@]}"; do
+  echo "Argument: $arg"
+done
+
+/usr/local/bin/wait-for-apiserver.sh
+
+echo "starting higress..."
+
+/usr/local/bin/higress "$@"
+
+echo "higress started."

docker/wait-for-apiserver.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#  Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       http:www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# if has HIGRESS_APISERVER_SVC env, wait for apiserver ready
+if [ -n "$HIGRESS_APISERVER_SVC" ]; then
+    while true; do
+        echo "testing higress apiserver is ready to connect..."
+        nc -z "$HIGRESS_APISERVER_SVC" "${HIGRESS_APISERVER_PORT}"
+        if [ $? -eq 0 ]; then
+            break
+        fi
+        sleep 1
+    done
+fi

helm/core/templates/_helpers.tpl

@@ -101,3 +101,7 @@ higress: {{ include "controller.name" . }}
 true
 {{- end }}
 {{- end }}
+
+{{- define "apiserver.name" -}}
+{{- .Values.apiserver.name | default "higress-apiserver" -}}
+{{- end }}

helm/core/templates/controller-deployment.yaml

@@ -189,6 +189,9 @@ spec:
           image: "{{ .Values.controller.hub | default .Values.global.hub }}/{{ .Values.controller.image | default "higress" }}:{{ .Values.controller.tag | default .Chart.AppVersion }}"
           args:
           - "serve"
+          {{- if .Values.apiserver.enabled }}
+          - --kubeconfig=/etc/istio/config/kubeconfig
+          {{- end }}
           - --gatewaySelectorKey=higress
           - --gatewaySelectorValue={{ .Release.Namespace }}-{{ include "gateway.name" . }}
           {{- if not .Values.global.enableStatus }}
@@ -199,6 +202,12 @@ spec:
           - --watchNamespace={{ .Values.global.watchNamespace }}
           {{- end }}
           env:
+          {{- if .Values.apiserver.enabled }}
+          - name: HIGRESS_APISERVER_SVC
+            value: {{ .Values.apiserver.addr | quote }}
+          - name: HIGRESS_APISERVER_PORT
+            value: {{ .Values.apiserver.securePort | quote }}
+          {{- end }}
           - name: POD_NAME
             valueFrom:
               fieldRef:
@@ -237,6 +246,96 @@ spec:
           volumeMounts:
           - name: log
             mountPath: /var/log
+          {{- if .Values.apiserver.enabled }}
+          - name: config
+            mountPath: /etc/istio/config
+          {{- end }}
+        {{- if .Values.apiserver.enabled }}
+        - name: {{ .Values.apiserver.name }}
+          image: "{{ .Values.apiserver.hub | default .Values.global.hub }}/{{ .Values.apiserver.image }}:{{ .Values.apiserver.tag }}"
+          imagePullPolicy: {{ .Values.apiserver.imagePullPolicy }}
+          args:
+          - --secure-port
+          - {{ .Values.apiserver.securePort | quote }}
+          - --client-ca-file
+          - /etc/api/ca.crt
+          - --tls-cert-file
+          - /etc/api/server.crt
+          - --tls-private-key-file
+          - /etc/api/server.key
+          - --storage
+          - {{ .Values.apiserver.storage }}
+          - --{{ .Values.apiserver.storage }}-server
+          - {{ .Values.apiserver.serverAddr }}
+          - --{{ .Values.apiserver.storage }}-username
+          - {{ .Values.apiserver.username }}
+          - --{{ .Values.apiserver.storage }}-password
+          - {{ .Values.apiserver.password }}
+          - --{{ .Values.apiserver.storage }}-ns-id
+          - {{ .Values.apiserver.namespaceID }}
+          - --{{ .Values.apiserver.storage }}-encryption-key-file
+          - /etc/api/nacos.key
+          env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+               fieldRef:
+                 apiVersion: v1
+                 fieldPath: metadata.namespace
+          - name: INSTANCE_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.podIP
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.hostIP
+          ports:
+          - containerPort: 8443
+            hostPort: 8443
+            name: https
+            protocol: TCP
+          readinessProbe:
+            failureThreshold: 30
+            httpGet:
+              path: /readyz
+              port: 8443
+              scheme: HTTPS
+            initialDelaySeconds: 1
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 3
+          resources: { }
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsGroup: 1337
+            runAsNonRoot: true
+            runAsUser: 1337
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/api
+              name: cert-config
+              readOnly: true
+            - mountPath: /tmp/nacos
+              name: nacos-data
+        {{- end }}
       {{- with .Values.controller.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -252,10 +351,28 @@ spec:
       volumes:
       - name: log
         emptyDir: {}
+      {{- if .Values.apiserver.enabled }}
+      - name: {{ .Values.apiserver.storage }}-data
+        emptyDir: {}
+      - name: cert-config
+        secret:
+          defaultMode: 420
+          secretName: higress-apiserver
+      {{- end }}
       {{- if not .Values.global.enableHigressIstio }}
       - name: config
+        {{- if not .Values.apiserver.enabled }}
         configMap:
           name: higress-config
+        {{- end }}
+        {{- if .Values.apiserver.enabled }}
+        projected:
+          sources:
+          - configMap:
+              name: higress-config
+          - configMap:
+              name: higress-apiserver
+        {{- end }}
       # Technically not needed on this pod - but it helps debugging/testing SDS
       # Should be removed after everything works.
       - emptyDir:

helm/core/templates/controller-service.yaml

@@ -23,6 +23,12 @@ spec:
     - port: 15014
       name: http-monitoring # prometheus stats
       protocol: TCP
+    {{- if .Values.apiserver.enabled }}
+    - port: {{ .Values.apiserver.securePort }}
+      name: https-apiserver
+      protocol: TCP
+      targetPort: {{ .Values.apiserver.securePort }}
+    {{- end }}
     {{- end }}
   selector:
-    {{- include "controller.selectorLabels" . | nindent 4 }}
+    {{- include "controller.selectorLabels" . | nindent 4 }}

helm/core/values.yaml

@@ -615,6 +615,23 @@ pilot:
   # Additional labels to apply on the pod level for monitoring and logging configuration.
   podLabels: {}
 
+## Higress ApiServer Storage Settings
+## Todo simplify this
+apiserver:
+  addr: 127.0.0.1
+  enabled: false
+  name: "higress-apiserver"
+  hub: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress
+  tag: 0.0.10
+  image: api-server
+  imagePullPolicy: IfNotPresent
+  securePort: 8443
+  storage: nacos
+  serverAddr: http://127.0.0.1:8848
+  username: ""
+  password: ""
+  namespaceID: ""
+  rsaKeyLength: 4096
 
 # Skywalking config settings
 skywalking: