Skip to main content

Karpenter


Prerequisites

진행 전 Spot Role에 대한 설정을 먼저 해주세요.

Karpenter Role

import * as aws from "@pulumi/aws";
import * as variable from "@src/variable";

function createControllerRole(): aws.iam.Role {
const name = "eks-karpenter-controller-role";
const oidcUrl = variable.eks.core.eks.apply((eks) => eks.oidcProvider.url);

return new aws.iam.Role(
name,
{
namePrefix: `${name}-`,
assumeRolePolicy: {
Version: "2012-10-17",
Statement: [
{
Action: "sts:AssumeRoleWithWebIdentity",
Effect: "Allow",
Principal: {
Federated: oidcUrl.apply(
(url) =>
`arn:aws:iam::${variable.awsAccountId}:oidc-provider/${url}`
),
},
Condition: {
StringEquals: oidcUrl.apply((url) => ({
[`${url}:sub`]: "system:serviceaccount:karpenter:karpenter",
})),
},
},
],
},
tags: {
Name: name,
"loliot.net/cluster": variable.clusterName,
"loliot.net/stack": variable.stackName,
},
},
{ protect: true }
);
}

export const controllerRole = createControllerRole();

function createControllerPolicy(): aws.iam.Policy {
const name = "eks-karpenter-controller-policy";
return new aws.iam.Policy(
name,
{
namePrefix: `${name}-`,
policy: {
Version: "2012-10-17",
Statement: [
{
Action: [
"ec2:CreateLaunchTemplate",
"ec2:CreateFleet",
"ec2:RunInstances",
"ec2:CreateTags",
"iam:PassRole",
"ec2:TerminateInstances",
"ec2:DescribeLaunchTemplates",
"ec2:DeleteLaunchTemplate",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeImages",
"ec2:DescribeInstances",
"ec2:DescribeInstanceTypes",
"ec2:DescribeInstanceTypeOfferings",
"ec2:DescribeAvailabilityZones",
"ssm:GetParameter",
"pricing:GetProducts",
],
Effect: "Allow",
Resource: "*",
},
],
},
tags: {
Name: name,
"loliot.net/cluster": variable.clusterName,
"loliot.net/stack": variable.stackName,
},
},
{ protect: true }
);
}

const controllerPolicy = createControllerPolicy();

new aws.iam.RolePolicyAttachment(
"eks-karpenter-rpa-0",
{
policyArn: controllerPolicy.arn,
role: controllerRole.name,
},
{ protect: true }
);

NodeGroup Role

function createNodeRole(): aws.iam.Role {
const name = "eks-karpenter-ng-role";
return new aws.iam.Role(
name,
{
namePrefix: `${name}-`,
assumeRolePolicy: {
Version: "2012-10-17",
Statement: [
{
Action: "sts:AssumeRole",
Effect: "Allow",
Principal: {
Service: "ec2.amazonaws.com",
},
},
],
},
tags: {
Name: name,
"loliot.net/cluster": variable.clusterName,
"loliot.net/stack": variable.stackName,
},
},
{ protect: true }
);
}

const nodeRole = createNodeRole();

function createInstanceProfile(): aws.iam.InstanceProfile {
const name = "eks-karpenter-ng-profile";
return new aws.iam.InstanceProfile(
name,
{
namePrefix: `${name}-`,
role: nodeRole.name,
tags: {
Name: name,
"loliot.net/cluster": variable.clusterName,
"loliot.net/stack": variable.stackName,
},
},
{ protect: true }
);
}

const instanceProfile = createInstanceProfile();

const nodePolicyARNs = {
"0": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
"1": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
"2": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
"3": "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", // Karpenter
};

const rpas = Object.entries(nodePolicyARNs).map(
([i, arn]) =>
new aws.iam.RolePolicyAttachment(
`eks-karpenter-ng-rpa-${i}`,
{
policyArn: arn,
role: nodeRole.name,
},
{ protect: true }
)
);

Installation

danger

설치 전에 CoreDNS가 작동하는 지 확인하시기 바랍니다.

helm show values oci://public.ecr.aws/karpenter/karpenter \
--version v0.19.1 \
> karpenter-values.yaml
additionalLabels: {}

serviceAccount:
annotations:
eks.amazonaws.com/role-arn: arn:aws:iam::<account_id>:role/<role_name>

replicas: 1

affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: karpenter.sh/provisioner-name
operator: DoesNotExist

tolerations:
- operator: Exists

settings:
# Pending Pod가 새로 생성되면 batch를 만들려고 시도함
# 새 Pending Pod가 생기고 batchIdleDuration 시간 만큼 대기
# 그 안에 새로운 Pending Pod가 생기면 batch에 포함 시키고 다시 batchIdleDuration 시간 대기
# 이 과정이 반복되다가 마지막 Pending Pod가 생긴 후 batchIdleDuration을 넘기거나
# 첫 Pending Pod가 생긴 후 batchMaxDuration을 넘기면 batch만들기 종료
# https://github.com/aws/karpenter/blob/main/pkg/controllers/provisioning/batcher.go
batchMaxDuration: 20s
batchIdleDuration: 5s

aws:
clusterName: <clusterName>

clusterEndpoint: <clusterEndpoint>

defaultInstanceProfile: <instanceProfile>
kubectl create namespace karpenter
helm upgrade karpenter oci://public.ecr.aws/karpenter/karpenter \
--install \
--version v0.19.1 \
-n karpenter \
--history-max 3 \
-f karpenter-values.yaml

Reference