feat: support nacos storage

.github/workflows/build-and-test-plugin.yaml

@@ -61,6 +61,43 @@ jobs:
     - name: "Run Ingress WasmPlugins Tests"
       run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} make higress-wasmplugin-test
 
+  higress-wasmplugin-test-nacos:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        wasmPluginType: [ GO ]
+        wasmPluginName: [ request-block ]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Setup Golang Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-go
+
+      - name: Setup Submodule Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            .git/modules
+          key: ${{ runner.os }}-submodules-cache-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-submodules-cache
+
+      - run: git stash # restore patch
+
+      - name: "Run Ingress WasmPlugins Tests"
+        run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} PLUGIN_NAME=${{ matrix.wasmPluginName }} make higress-wasmplugin-test
+
   publish:
     runs-on: ubuntu-latest
     needs: [higress-wasmplugin-test]

.github/workflows/build-and-test.yaml

@@ -134,7 +134,41 @@ jobs:
 
     - name: "Run Higress E2E Conformance Tests"
       run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test
-
+
+  higress-conformance-test-nacos:
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Setup Golang Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-go
+
+      - name: Setup Submodule Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            .git/modules
+          key: ${{ runner.os }}-submodules-cache-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-submodules-cache
+
+      - run: git stash # restore patch
+
+      - name: "Run Higress E2E Conformance Nacos Tests"
+        run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test-nacos
+
   publish:
     runs-on: ubuntu-latest
     needs: [higress-conformance-test,gateway-conformance-test]

Makefile.core.mk

@@ -184,6 +184,14 @@ install-dev: pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'
 install-dev-wasmplugin: build-wasmplugins pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'  --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false'
+install-dev-nacos: pre-install
+	kubectl apply -f tools/hack/conf/nacos.yaml
+	tools/hack/gen-secret-configmap.sh
+	helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"'
+install-dev-wasmplugin-nacos: build-wasmplugins pre-install
+	kubectl apply -f tools/hack/conf/nacos.yaml
+	tools/hack/gen-secret-configmap.sh
+	helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false'
 
 uninstall:
 	helm uninstall higress -n higress-system
@@ -257,6 +265,19 @@ higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-buil
 .PHONY: higress-wasmplugin-test-clean
 higress-wasmplugin-test-clean: $(tools/kind) delete-cluster
 
+# higress-conformance-test-nacos runs ingress nacos tests.
+.PHONY: higress-conformance-test-nacos
+higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-nacos delete-cluster
+
+# higress-wasmplugin-test-nacos runs ingress wasmplugin nacos tests.
+.PHONY: higress-wasmplugin-test-nacos
+higress-wasmplugin-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin-nacos port-forward-nacos run-higress-e2e-test-wasmplugin-nacos delete-cluster
+
+.PHONY: port-forward-nacos
+port-forward-nacos:
+	kubectl wait --timeout=10m -n higress-system deploy/nacos --for=condition=Available
+	kubectl port-forward -n higress-system svc/nacos-service 8848:8848 9848:9848 &
+
 # create-cluster creates a kube cluster with kind.
 .PHONY: create-cluster
 create-cluster: $(tools/kind)
@@ -275,13 +296,13 @@ delete-cluster: $(tools/kind) ## Delete kind cluster.
 kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG.
 	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)
 	tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
-	tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
+	tools/hack/docker-pull-image.sh docker.io/swsk33/nacos-standalone 2.2.3
 	tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0
 	tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
 	tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest
 	tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2
 	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
-	tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
+	tools/hack/kind-load-image.sh docker.io/swsk33/nacos-standalone 2.2.3
 	tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0
 	tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2
 	tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
@@ -346,8 +367,8 @@ run-higress-e2e-test-wasmplugin:
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
 	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=all
-
-# run-higress-e2e-test-wasmplugin-run starts to run ingress e2e conformance tests.
+  
+ # run-higress-e2e-test-wasmplugin-run starts to run ingress e2e conformance tests.
 .PHONY: run-higress-e2e-test-wasmplugin-run
 run-higress-e2e-test-wasmplugin-run:
 	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
@@ -356,8 +377,8 @@ run-higress-e2e-test-wasmplugin-run:
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
 	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=run
-
-# run-higress-e2e-test-wasmplugin-clean starts to clean ingress e2e tests.
+  
+ # run-higress-e2e-test-wasmplugin-clean starts to clean ingress e2e tests.
 .PHONY: run-higress-e2e-test-wasmplugin-clean
 run-higress-e2e-test-wasmplugin-clean:
 	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
@@ -366,3 +387,22 @@ run-higress-e2e-test-wasmplugin-clean:
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
 	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=clean
+
+.PHONY: run-higress-e2e-test-nacos
+run-higress-e2e-test-nacos:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true
+
+.PHONY: run-higress-e2e-test-wasmplugin-nacos
+run-higress-e2e-test-wasmplugin-nacos:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --enableApiServer=true
+

docker/Dockerfile.higress

@@ -13,7 +13,11 @@ FROM ${HUB}/base:${BASE_VERSION}
 ARG TARGETARCH
 
 COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress
+COPY wait-for-apiserver.sh /usr/local/bin/wait-for-apiserver.sh
+COPY run-higress.sh /usr/local/bin/run-higress.sh
+
+RUN chmod +x /usr/local/bin/*.sh
 
 USER 1337:1337
 
-ENTRYPOINT ["/usr/local/bin/higress"]
+ENTRYPOINT ["/usr/local/bin/run-higress.sh"]

docker/docker.mk

@@ -14,12 +14,16 @@
 
 docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress: $(OUT_LINUX)/higress
+docker.higress: docker/wait-for-apiserver.sh
+docker.higress: docker/run-higress.sh
 docker.higress: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_RULE)
 
 docker.higress-buildx: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress-buildx: $(AMD64_OUT_LINUX)/higress
 docker.higress-buildx: $(ARM64_OUT_LINUX)/higress
+docker.higress: docker/wait-for-apiserver.sh
+docker.higress: docker/run-higress.sh
 docker.higress-buildx: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_BUILDX_RULE)
 

docker/run-higress.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#  Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       http:www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+args=("$@")
+
+for arg in "${args[@]}"; do
+  echo "Argument: $arg"
+done
+
+/usr/local/bin/wait-for-apiserver.sh
+
+echo "starting higress..."
+
+exec /usr/local/bin/higress "$@"

docker/wait-for-apiserver.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#  Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       http:www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# if has HIGRESS_APISERVER_SVC env, wait for apiserver ready
+if [ -n "$HIGRESS_APISERVER_SVC" ]; then
+    while true; do
+        echo "testing higress apiserver is ready to connect..."
+        nc -z "$HIGRESS_APISERVER_SVC" "${HIGRESS_APISERVER_PORT}"
+        if [ $? -eq 0 ]; then
+            break
+        fi
+        sleep 1
+    done
+fi

helm/core/templates/_helpers.tpl

@@ -101,3 +101,7 @@ higress: {{ include "controller.name" . }}
 true
 {{- end }}
 {{- end }}
+
+{{- define "apiserver.name" -}}
+{{- .Values.apiserver.name | default "higress-apiserver" -}}
+{{- end }}