alibaba · sjcsjc123 · Jan 22, 2024 · Jan 22, 2024 · Jan 22, 2024 · Jan 22, 2024
diff --git a/Makefile.core.mk b/Makefile.core.mk
@@ -184,6 +184,13 @@ install-dev: pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'
 install-dev-wasmplugin: build-wasmplugins pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'  --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false'
+install-dev-nacos: pre-install
+	kubectl apply -f tools/hack/conf/nacos.yaml
+	tools/hack/gen-keys.sh
+	helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'apiserver.serverAddr="http://nacos-service.higress-system.svc.cluster.local:8848"'
+
+create-nacos:
+	kubectl apply -f tools/hack/conf/nacos.yaml
 
 uninstall:
 	helm uninstall higress -n higress-system
@@ -258,18 +265,18 @@ delete-cluster: $(tools/kind) ## Delete kind cluster.
 .PHONY: kube-load-image
 kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG.
 	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)
-	tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
-	tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
-	tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0
-	tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
-	tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest
-	tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2
-	tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
-	tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
-	tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0
-	tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2
-	tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
-	tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest
+	#tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
+	#tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
+	#tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0
+	#tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
+	#tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest
+	#tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2
+	#tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86
+	#tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3
+	#tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0
+	#tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2
+	#tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0
+	#tools/hack/kind-load-image.sh docker.io/bitinit/eureka latest
 # run-higress-e2e-test starts to run ingress e2e tests.
 .PHONY: run-higress-e2e-test
 run-higress-e2e-test:

diff --git a/docker/Dockerfile.higress b/docker/Dockerfile.higress
@@ -13,7 +13,12 @@ FROM ${HUB}/base:${BASE_VERSION}
 ARG TARGETARCH
 
 COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress
+COPY wait-for-apiserver.sh /usr/local/bin/wait-for-apiserver.sh
+COPY run-higress.sh /usr/local/bin/run-higress.sh
+
+RUN chmod +x /usr/local/bin/wait-for-apiserver.sh
+RUN chmod +x /usr/local/bin/run-higress.sh
 
 USER 1337:1337
 
-ENTRYPOINT ["/usr/local/bin/higress"]
+ENTRYPOINT ["/usr/local/bin/run-higress.sh"]
diff --git a/docker/docker.mk b/docker/docker.mk
@@ -14,12 +14,16 @@
 
 docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress: $(OUT_LINUX)/higress
+docker.higress: docker/wait-for-apiserver.sh
+docker.higress: docker/run-higress.sh
 docker.higress: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_RULE)
 
 docker.higress-buildx: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB}
 docker.higress-buildx: $(AMD64_OUT_LINUX)/higress
 docker.higress-buildx: $(ARM64_OUT_LINUX)/higress
+docker.higress: docker/wait-for-apiserver.sh
+docker.higress: docker/run-higress.sh
 docker.higress-buildx: docker/Dockerfile.higress
 	$(HIGRESS_DOCKER_BUILDX_RULE)
 

diff --git a/docker/run-higress.sh b/docker/run-higress.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#  Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       http:www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+args=("$@")
+
+for arg in "${args[@]}"; do
+  echo "Argument: $arg"
+done
+
+/usr/local/bin/wait-for-apiserver.sh
+
+echo "starting higress..."
+
+/usr/local/bin/higress "$@"
+
+echo "higress started."
diff --git a/docker/wait-for-apiserver.sh b/docker/wait-for-apiserver.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#  Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+
+#       http:www.apache.org/licenses/LICENSE-2.0
+
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# if has HIGRESS_APISERVER_SVC env, wait for apiserver ready
+if [ -n "$HIGRESS_APISERVER_SVC" ]; then
+    while true; do
+        echo "testing higress apiserver is ready to connect..."
+        nc -z "$HIGRESS_APISERVER_SVC" "${HIGRESS_APISERVER_PORT}"
+        if [ $? -eq 0 ]; then
+            break
+        fi
+        sleep 1
+    done
+fi
diff --git a/helm/core/templates/_helpers.tpl b/helm/core/templates/_helpers.tpl
@@ -101,3 +101,7 @@ higress: {{ include "controller.name" . }}
 true
 {{- end }}
 {{- end }}
+
+{{- define "apiserver.name" -}}
+{{- .Values.apiserver.name | default "higress-apiserver" -}}
+{{- end }}
diff --git a/helm/core/templates/configmap.yaml b/helm/core/templates/configmap.yaml
@@ -105,7 +105,28 @@ metadata:
   labels:
     {{- include "gateway.labels" . | nindent 4 }}    
 data:
-
+  {{- if .Values.apiserver.enabled }}
+  kubeconfig: |
+    apiVersion: v1
+    kind: Config
+    clusters:
+      - name: {{ .Release.Namespace }}
+        cluster:
+          server: https://{{ .Values.apiserver.addr }}:{{ .Values.apiserver.securePort }}
+          insecure-skip-tls-verify: true
+    users:
+      - name: higress-admin
+        user:
+          client-certificate-data: {{ .Files.Get "files/api/client.crt" | b64enc }}
+          client-key-data: {{ .Files.Get "files/api/client.key" | b64enc }}
+    contexts:
+      - name: higress
+        context:
+          cluster: higress
+          user: higress-admin
+    preferences: {}
+    current-context: higress
+  {{- end }}
   # Configuration file for the mesh networks to be used by the Split Horizon EDS.
   meshNetworks: |-
   {{- if .Values.global.meshNetworks }}

diff --git a/helm/core/templates/controller-deployment.yaml b/helm/core/templates/controller-deployment.yaml
@@ -189,6 +189,9 @@ spec:
           image: "{{ .Values.controller.hub | default .Values.global.hub }}/{{ .Values.controller.image | default "higress" }}:{{ .Values.controller.tag | default .Chart.AppVersion }}"
           args:
           - "serve"
+          {{- if .Values.apiserver.enabled }}
+          - --kubeconfig=/etc/istio/config/kubeconfig
+          {{- end }}
           - --gatewaySelectorKey=higress
           - --gatewaySelectorValue={{ .Release.Namespace }}-{{ include "gateway.name" . }}
           {{- if not .Values.global.enableStatus }}
@@ -199,6 +202,12 @@ spec:
           - --watchNamespace={{ .Values.global.watchNamespace }}
           {{- end }}
           env:
+          {{- if .Values.apiserver.enabled }}
+          - name: HIGRESS_APISERVER_SVC
+            value: {{ .Values.apiserver.addr | quote }}
+          - name: HIGRESS_APISERVER_PORT
+            value: {{ .Values.apiserver.securePort | quote }}
+          {{- end }}
           - name: POD_NAME
             valueFrom:
               fieldRef:
@@ -237,6 +246,96 @@ spec:
           volumeMounts:
           - name: log
             mountPath: /var/log
+          {{- if .Values.apiserver.enabled }}
+          - name: config
+            mountPath: /etc/istio/config
+          {{- end }}
+        {{- if .Values.apiserver.enabled }}
+        - name: {{ .Values.apiserver.name }}
+          image: "{{ .Values.apiserver.hub | default .Values.global.hub }}/{{ .Values.apiserver.image }}:{{ .Values.apiserver.tag }}"
+          imagePullPolicy: {{ .Values.apiserver.imagePullPolicy }}
+          args:
+          - --secure-port
+          - {{ .Values.apiserver.securePort | quote }}
+          - --client-ca-file
+          - /etc/api/ca.crt
+          - --tls-cert-file
+          - /etc/api/server.crt
+          - --tls-private-key-file
+          - /etc/api/server.key
+          - --storage
+          - {{ .Values.apiserver.storage }}
+          - --{{ .Values.apiserver.storage }}-server
+          - {{ .Values.apiserver.serverAddr }}
+          - --{{ .Values.apiserver.storage }}-username
+          - {{ .Values.apiserver.username }}
+          - --{{ .Values.apiserver.storage }}-password
+          - {{ .Values.apiserver.password }}
+          - --{{ .Values.apiserver.storage }}-ns-id
+          - {{ .Values.apiserver.namespaceID }}
+          - --{{ .Values.apiserver.storage }}-encryption-key-file
+          - /etc/api/nacos.key
+          env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+               fieldRef:
+                 apiVersion: v1
+                 fieldPath: metadata.namespace
+          - name: INSTANCE_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.podIP
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: status.hostIP
+          ports:
+          - containerPort: 8443
+            hostPort: 8443
+            name: https
+            protocol: TCP
+          readinessProbe:
+            failureThreshold: 30
+            httpGet:
+              path: /readyz
+              port: 8443
+              scheme: HTTPS
+            initialDelaySeconds: 1
+            periodSeconds: 2
+            successThreshold: 1
+            timeoutSeconds: 3
+          resources: { }
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            privileged: false
+            readOnlyRootFilesystem: true
+            runAsGroup: 1337
+            runAsNonRoot: true
+            runAsUser: 1337
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+          volumeMounts:
+            - mountPath: /etc/api
+              name: cert-config
+              readOnly: true
+            - mountPath: /tmp/nacos
+              name: nacos-data
+        {{- end }}
       {{- with .Values.controller.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -252,6 +351,14 @@ spec:
       volumes:
       - name: log
         emptyDir: {}
+      {{- if .Values.apiserver.enabled }}
+      - name: {{ .Values.apiserver.storage }}-data
+        emptyDir: {}
+      - name: cert-config
+        secret:
+          defaultMode: 420
+          secretName: higress-apiserver
+      {{- end }}
       {{- if not .Values.global.enableHigressIstio }}
       - name: config
         configMap:

diff --git a/helm/core/templates/controller-secret.yaml b/helm/core/templates/controller-secret.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.apiserver.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+  name: {{ include "apiserver.name" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  ca.crt: {{ .Files.Get "files/api/ca.crt" | b64enc }}
+  ca.key: {{ .Files.Get "files/api/ca.key" | b64enc }}
+  server.crt: {{ .Files.Get "files/api/server.crt" | b64enc }}
+  server.key: {{ .Files.Get "files/api/server.key" | b64enc }}
+  client.key: {{ .Files.Get "files/api/client.crt" | b64enc }}
+  nacos.key: {{ .Files.Get "files/api/nacos.key" | b64enc }}
+{{- end }}
diff --git a/helm/core/templates/controller-service.yaml b/helm/core/templates/controller-service.yaml
@@ -23,6 +23,12 @@ spec:
     - port: 15014
       name: http-monitoring # prometheus stats
       protocol: TCP
+    {{- if .Values.apiserver.enabled }}
+    - port: {{ .Values.apiserver.securePort }}
+      name: https-apiserver # mTLS with k8s-signed cert
+      protocol: TCP
+      targetPort: {{ .Values.apiserver.securePort }}
+    {{- end }}
     {{- end }}
   selector:
-    {{- include "controller.selectorLabels" . | nindent 4 }}
+    {{- include "controller.selectorLabels" . | nindent 4 }}
diff --git a/helm/core/values.yaml b/helm/core/values.yaml
@@ -615,6 +615,24 @@ pilot:
   # Additional labels to apply on the pod level for monitoring and logging configuration.
   podLabels: {}
 
+## Higress ApiServer Storage Settings
+## Todo simplify this
+apiserver:
+  addr: 127.0.0.1
+  enabled: false
+  name: "higress-apiserver"
+  hub: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress
+  tag: 0.0.10
+  image: api-server
+  imagePullPolicy: IfNotPresent
+  securePort: 8443
+  storage: nacos
+  serverAddr: http://127.0.0.1:8848
+  username: ""
+  password: ""
+  namespaceID: ""
+  rsaKeyLength: 4096
+  serviceName: "higress-controller"
 
 # Skywalking config settings
 skywalking: