apiVersion: k8s.cni.cncf.io/v1beta1
kind: MultiNetworkPolicy
metadata:
  name: test-network-policy
  namespace: default
  annotations:
    k8s.v1.cni.cncf.io/policy-for: bridge-network
spec:
  podSelector:
    matchLabels:
      app: debug
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - ipBlock:
        cidr: 10.10.0.5/24
  egress:
  - to:
    - ipBlock:
        cidr: 10.10.0.7/32

[142:11928] -A INPUT -i net1 -j MULTI-INGRESS
[478:40152] -A OUTPUT -o net1 -j MULTI-EGRESS
[0:0] -A MULTI-0-EGRESS -j MARK --set-xmark 0x0/0x30000
[0:0] -A MULTI-0-EGRESS -j MULTI-0-EGRESS-0-PORTS
[0:0] -A MULTI-0-EGRESS -j MULTI-0-EGRESS-0-TO
[0:0] -A MULTI-0-EGRESS -m mark --mark 0x30000/0x30000 -j RETURN
[0:0] -A MULTI-0-EGRESS -j DROP
[0:0] -A MULTI-0-EGRESS-0-PORTS -m comment --comment "no egress ports, skipped" -j MARK --set-xmark 0x10000/0x10000
[0:0] -A MULTI-0-EGRESS-0-TO -d 10.10.0.7/32 -o net1 -j MARK --set-xmark 0x20000/0x20000
[0:0] -A MULTI-0-INGRESS -j MARK --set-xmark 0x0/0x30000
[0:0] -A MULTI-0-INGRESS -j MULTI-0-INGRESS-0-PORTS
[0:0] -A MULTI-0-INGRESS -j MULTI-0-INGRESS-0-FROM
[0:0] -A MULTI-0-INGRESS -m mark --mark 0x30000/0x30000 -j RETURN
[0:0] -A MULTI-0-INGRESS -j DROP
[0:0] -A MULTI-0-INGRESS-0-FROM -s 10.10.0.0/24 -i net1 -j MARK --set-xmark 0x20000/0x20000
[0:0] -A MULTI-0-INGRESS-0-PORTS -m comment --comment "no ingress ports, skipped" -j MARK --set-xmark 0x10000/0x10000
[0:0] -A MULTI-EGRESS -o net1 -m comment --comment "policy:test-network-policy net-attach-def:default/bridge-network" -j MULTI-0-EGRESS
[0:0] -A MULTI-INGRESS -i net1 -m comment --comment "policy:test-network-policy net-attach-def:default/bridge-network" -j MULTI-0-INGRESS
COMMIT

$ cat kustomization.yaml
helmCharts:
- name: redis-cluster
  includeCRDs: false
  valuesFile: redis.yaml
  releaseName: redis-cluster
  namespace: production
  version: 9.0.5
  repo: https://charts.bitnami.com/bitnami

$ cat redis.yaml

global:
  storageClass: standard-rwo
existingSecret: redis-cluster
existingSecretPasswordKey: password
redis:
  resources:
    limits:
      cpu: 2
      memory: 2Gi
    requests:
      cpu: 0.1
      memory: 0.5Gi
  podAnnotations:
    'xxxxx': 'yyyyy'

usePassword: false

redis-cluster                       ClusterIP      10.2.5.248    <none>        6379/TCP                                                                                          213d
redis-cluster-headless              ClusterIP      None          <none>        6379/TCP,16379/TCP                                                                                213d

$ kubectl port-forward --address 0.0.0.0 pod/redis-cluster-0 6379:6379
Forwarding from 0.0.0.0:6379 -> 6379
$ redis-benchmark -n 1 --cluster
Cluster has 3 master nodes:

Master 0: 1020525d25c7ad16e786a98e1eb7419d609b8847 10.4.0.119:6379
Could not connect to Redis at 10.4.0.119:6379: Connection refused

$ kubectl run --image=hwchiu/netutils test
$ kubectl exec -it test -- bash
root@test:/# redis-benchmark -h redis-cluster -q  -c 20 -n 30000
PING_INLINE: 44977.51 requests per second
PING_BULK: 48154.09 requests per second
SET: 45317.22 requests per second
GET: 47169.81 requests per second
INCR: 50251.26 requests per second
LPUSH: 48465.27 requests per second
LPOP: 41265.48 requests per second
SADD: 37878.79 requests per second
SPOP: 49833.89 requests per second
LPUSH (needed to benchmark LRANGE): 51724.14 requests per second
LRANGE_100 (first 100 elements): 43041.61 requests per second
LRANGE_300 (first 300 elements): 35842.29 requests per second
LRANGE_500 (first 450 elements): 36014.41 requests per second
LRANGE_600 (first 600 elements): 33259.42 requests per second
MSET (10 keys): 42796.01 requests per second

{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }}
{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy" }}
kind: ClusterRole
...
{{- end }}
{{- end }}

...
  spec:
    values:
      global:
        proxy:
          privileged: false
          readinessFailureThreshold: 30
          readinessInitialDelaySeconds: 1
          readinessPeriodSeconds: 2
          resources:
            limits:
              cpu: 2000m
              memory: 1024Mi
            requests:
              cpu: 100m
              memory: 128Mi

annotations:
  sidecar.istio.io/proxyCPU: 50m
  sidecar.istio.io/proxyMemory: 128Mi

istio.io/rev=1-14-2

$ terraform apply
Acquiring state lock. This may take a few moments...
╷
│ Error: Error acquiring the state lock
│
│ Error message: writing "gs://xxxxx/xxxxxxx/default.tflock" failed: googleapi: Error 412: At least one of the pre-conditions you specified did not hold., conditionNotMet
│ Lock Info:
│   ID:        1696991555387294
│   Path:      gs://xxxxx/xxxxxxx/default.tflock
│   Operation: OperationTypeApply
│   Who:       your_name@hostname.local
│   Version:   1.5.6
│   Created:   2022-10-11 02:32:35.12734 +0000 UTC
│   Info:
│
│ Terraform acquires a state lock to protect the state from being written
│ by multiple users at the same time. Please resolve the issue above and try
│ again. For most commands, you can disable locking with the "-lock=false"
│ flag, but this is not recommended.

$ terraform force-unlock 1696991555387294
Do you really want to force-unlock?
  Terraform will remove the lock on the remote state.
  This will allow local Terraform commands to modify this state, even though it
  may still be in use. Only 'yes' will be accepted to confirm.

  Enter a value: yes

terraform state replace-provider "registry.terraform.io/-/aws" "hashicorp/aws"

    cluster-autoscaler.kubernetes.io/safe-to-evict: 'false'

$ kc explain pod.spec.terminationGracePeriodSeconds
KIND:     Pod
VERSION:  v1

FIELD:    terminationGracePeriodSeconds <integer>

DESCRIPTION:
     Optional duration in seconds the pod needs to terminate gracefully. May be
     decreased in delete request. Value must be non-negative integer. The value
     zero indicates stop immediately via the kill signal (no opportunity to shut
     down). If this value is nil, the default grace period will be used instead.
     The grace period is the duration in seconds after the processes running in
     the pod are sent a termination signal and the time when the processes are
     forcibly halted with a kill signal. Set this value longer than the expected
     cleanup time for your process. Defaults to 30 seconds.

function dump_secret {
  for i in $(kubectl -n $1 get --no-headers secret | awk '{print $1}'); do
    kubectl -n $1 get secret -o yaml $i > $i.yaml;
  done
}

function dump_secrets {
  for i in $(kubectl get ns --no-headers | awk '{print $1}'); do
      if [ ! -d "./$i" ]; then
          mkdir $i
      fi
      echo "Dump $i"
      cd $i
      dump_secret $i
      cd ..
  done
}

count_over_time({container="kube-event-exporter"}[1m] | json | __error__ != "JSONParserErr" | reason="OOMKilling")

sudo apt install libvirt-clients

$ virt-host-validate qemu
  QEMU: Checking for hardware virtualization                                 : PASS
  QEMU: Checking if device /dev/kvm exists                                   : PASS
  QEMU: Checking if device /dev/kvm is accessible                            : FAIL (Check /dev/kvm is world writable or you are in a group that is allowed to access it)
  QEMU: Checking if device /dev/vhost-net exists                             : PASS
  QEMU: Checking if device /dev/net/tun exists                               : PASS
  QEMU: Checking for cgroup 'cpu' controller support                         : PASS
  QEMU: Checking for cgroup 'cpuacct' controller support                     : PASS
  QEMU: Checking for cgroup 'cpuset' controller support                      : PASS
  QEMU: Checking for cgroup 'memory' controller support                      : PASS
  QEMU: Checking for cgroup 'devices' controller support                     : WARN (Enable 'devices' in kernel Kconfig file or mount/enable cgroup controller in your system)
  QEMU: Checking for cgroup 'blkio' controller support                       : PASS
  QEMU: Checking for device assignment IOMMU support                         : WARN (No ACPI DMAR table found, IOMMU either disabled in BIOS or not supported by this hardware platform)
  QEMU: Checking for secure guest support                                    : WARN (Unknown if this platform has Secure Guest support)

$ virt-host-validate qemu
  QEMU: Checking for hardware virtualization                                 : PASS
  QEMU: Checking if device /dev/kvm exists                                   : PASS
  QEMU: Checking if device /dev/kvm is accessible                            : PASS
  QEMU: Checking if device /dev/vhost-net exists                             : PASS
  QEMU: Checking if device /dev/net/tun exists                               : PASS
  QEMU: Checking for cgroup 'cpu' controller support                         : PASS
  QEMU: Checking for cgroup 'cpuacct' controller support                     : PASS
  QEMU: Checking for cgroup 'cpuset' controller support                      : PASS
  QEMU: Checking for cgroup 'memory' controller support                      : PASS
  QEMU: Checking for cgroup 'devices' controller support                     : WARN (Enable 'devices' in kernel Kconfig file or mount/enable cgroup controller in your system)
  QEMU: Checking for cgroup 'blkio' controller support                       : PASS
  QEMU: Checking for device assignment IOMMU support                         : WARN (No ACPI DMAR table found, IOMMU either disabled in BIOS or not supported by this hardware platform)
  QEMU: Checking for secure guest support                                    : WARN (Unknown if this platform has Secure Guest support)

$ minikube start --cni=flannel

$ export VERSION=$(curl -s https://api.github.com/repos/kubevirt/kubevirt/releases | grep tag_name | grep -v -- '-rc' | sort -r | head -1 | awk -F': ' '{print $2}' | sed 's/,//' | xargs)
$ echo $VERSION
$ kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml
namespace/kubevirt created
customresourcedefinition.apiextensions.k8s.io/kubevirts.kubevirt.io created
priorityclass.scheduling.k8s.io/kubevirt-cluster-critical created
clusterrole.rbac.authorization.k8s.io/kubevirt.io:operator created
serviceaccount/kubevirt-operator created
role.rbac.authorization.k8s.io/kubevirt-operator created
rolebinding.rbac.authorization.k8s.io/kubevirt-operator-rolebinding created
clusterrole.rbac.authorization.k8s.io/kubevirt-operator created
clusterrolebinding.rbac.authorization.k8s.io/kubevirt-operator created
deployment.apps/virt-operator created

$ kubectl -n kubevirt get pods
NAME                               READY   STATUS    RESTARTS   AGE
virt-operator-57f9fb965d-5lnqf     1/1     Running   0          46m
virt-operator-57f9fb965d-f5zg4     1/1     Running   0          46m

$ kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml

$ kubectl -n kubevirt get kv kubevirt -o yaml
apiVersion: kubevirt.io/v1
kind: KubeVirt
metadata:
  annotations:
    kubevirt.io/latest-observed-api-version: v1
    kubevirt.io/storage-observed-api-version: v1
  creationTimestamp: "2023-10-10T14:35:55Z"
  finalizers:
  - foregroundDeleteKubeVirt
  generation: 2
  name: kubevirt
  namespace: kubevirt
  resourceVersion: "1490"
  uid: bc621d93-4910-4b1f-b3c8-f8f1f4e27a38
spec:
  certificateRotateStrategy: {}
  configuration:
    developerConfiguration: {}
  customizeComponents: {}
  imagePullPolicy: IfNotPresent                                                                                                                                                                  workloadUpdateStrategy: {}
$ kubectl -n kubevirt get pods
NAME                               READY   STATUS    RESTARTS   AGE
virt-api-77f8d679fc-hntws          1/1     Running   0          49m
virt-controller-6689488456-4jtv8   1/1     Running   0          48m
virt-controller-6689488456-68hnz   1/1     Running   0          48m
virt-handler-psc4w                 1/1     Running   0          48m

VERSION=$(kubectl get kubevirt.kubevirt.io/kubevirt -n kubevirt -o=jsonpath="{.status.observedKubeVirtVersion}")
ARCH=$(uname -s | tr A-Z a-z)-$(uname -m | sed 's/x86_64/amd64/') || windows-amd64.exe
echo ${ARCH}
curl -L -o virtctl https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/virtctl-${VERSION}-${ARCH}
chmod +x virtctl
sudo install virtctl /usr/local/bin

-> % virtctl version
Client Version: version.Info{GitVersion:"v1.1.0-alpha.0", GitCommit:"67902ed9de43d7a0b94aa72b8fd7f48f31ca4285", GitTreeState:"clean", BuildDate:"2023-09-18T10:45:14Z", GoVersion:"go1.19.9", Compiler:"gc", Platform:"darwin/arm64"}
Server Version: version.Info{GitVersion:"v1.1.0-alpha.0", GitCommit:"67902ed9de43d7a0b94aa72b8fd7f48f31ca4285", GitTreeState:"clean", BuildDate:"2023-09-18T12:03:45Z", GoVersion:"go1.19.9", Compiler:"gc", Platform:"linux/arm64"}

$ kubectl apply -f https://kubevirt.io/labs/manifests/vm.yaml
virtualmachine.kubevirt.io/testvm created
$ kubectl get vm
NAME     AGE   STATUS    READY
testvm   7s    Stopped   False

$ virtctl start testvm
VM testvm was scheduled to start

$ kubectl get pods -o wide

$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast qlen 1000
    link/ether 52:54:00:0c:00:55 brd ff:ff:ff:ff:ff:ff
    inet 10.0.2.2/24 brd 10.0.2.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::5054:ff:fe0c:55/64 scope link
       valid_lft forever preferred_lft forever
$ ip r
default via 10.0.2.1 dev eth0
10.0.2.0/24 dev eth0  src 10.0.2.2

$ kubectl exec -it virt-launcher-testvm-pnn4j -- bash
bash-5.1$ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: eth0@if14: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
    link/ether 12:37:77:cf:6d:63 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 10.244.0.10/24 brd 10.244.0.255 scope global eth0
       valid_lft forever preferred_lft forever
3: k6t-eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default qlen 1000
    link/ether 02:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
    inet 10.0.2.1/24 brd 10.0.2.255 scope global k6t-eth0
       valid_lft forever preferred_lft forever
4: tap0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc fq_codel master k6t-eth0 state UP group default qlen 1000
    link/ether d6:0e:c5:6f:41:f1 brd ff:ff:ff:ff:ff:ff
bash-5.1$

bash-5.1$ ls /sys/class/net/k6t-eth0/brif
tap0
bash-5.1$ ls /sys/class/net/k6t-eth0/bridge/
ageing_time    group_fwd_mask          multicast_last_member_count     multicast_query_response_interval  nf_call_arptables   root_port                 vlan_protocol
bridge_id      hash_elasticity         multicast_last_member_interval  multicast_query_use_ifaddr         nf_call_ip6tables   stp_state                 vlan_stats_enabled
default_pvid   hash_max                multicast_membership_interval   multicast_router                   nf_call_iptables    tcn_timer                 vlan_stats_per_port
flush          hello_time              multicast_mld_version           multicast_snooping                 no_linklocal_learn  topology_change
forward_delay  hello_timer             multicast_querier               multicast_startup_query_count      priority            topology_change_detected
gc_timer       max_age                 multicast_querier_interval      multicast_startup_query_interval   root_id             topology_change_timer
group_addr     multicast_igmp_version  multicast_query_interval        multicast_stats_enabled            root_path_cost      vlan_filtering
bash-5.1$