mirror of
https://github.com/whyour/qinglong.git
synced 2026-09-22 19:29:13 +08:00
feat(ql3): establish 3.0 incubation baseline
This commit is contained in:
whyour
@@ -0,0 +1,568 @@
|
||||
const assert = require('node:assert/strict');
|
||||
const { test } = require('node:test');
|
||||
const {
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
workerCredentialDeliveryTokenDigest,
|
||||
} = require('@qinglong/runtime-core/worker-credential-delivery');
|
||||
const {
|
||||
formatWorkerCredentialToken,
|
||||
} = require('@qinglong/runtime-core/worker-credential-token');
|
||||
const {
|
||||
MAX_WORKER_CREDENTIAL_KUBERNETES_STAGES,
|
||||
WorkerCredentialKubernetesDeliveryAdapter,
|
||||
workerCredentialKubernetesDeploymentTargetDigest,
|
||||
} = require('@qinglong/cluster-admin/worker-credential-kubernetes-delivery');
|
||||
|
||||
const IDS = [
|
||||
'123e4567-e89b-42d3-a456-426614174901',
|
||||
'123e4567-e89b-42d3-a456-426614174902',
|
||||
'123e4567-e89b-42d3-a456-426614174903',
|
||||
];
|
||||
|
||||
function apiError(code) {
|
||||
return Object.assign(new Error(`Kubernetes API ${code}`), { code });
|
||||
}
|
||||
|
||||
function copy(value) {
|
||||
return structuredClone(value);
|
||||
}
|
||||
|
||||
class FakeKubernetesSecretApi {
|
||||
constructor() {
|
||||
this.items = new Map();
|
||||
this.deployments = new Map();
|
||||
this.revision = 0;
|
||||
this.uid = 0;
|
||||
this.failAfterCreate = false;
|
||||
this.failAfterReplace = false;
|
||||
this.failBeforeDeploymentReplace = false;
|
||||
this.failAfterDeploymentReplace = false;
|
||||
this.failAfterDelete = false;
|
||||
this.replacements = [];
|
||||
this.deploymentReplacements = [];
|
||||
this.deletions = [];
|
||||
const preparedTarget = this.serverSecret('qinglong-workers', {
|
||||
apiVersion: 'v1',
|
||||
kind: 'Secret',
|
||||
type: 'Opaque',
|
||||
metadata: {
|
||||
name: 'edge-router-1-credential',
|
||||
annotations: {
|
||||
'kubectl.kubernetes.io/last-applied-configuration':
|
||||
'{"kind":"Secret","metadata":{"name":"edge-router-1-credential"}}',
|
||||
},
|
||||
labels: {
|
||||
'app.kubernetes.io/managed-by': 'qinglong3',
|
||||
'qinglong.io/worker-credential-target': 'prepared-v3',
|
||||
},
|
||||
},
|
||||
data: {},
|
||||
}, null);
|
||||
this.items.set(
|
||||
this.key('qinglong-workers', 'edge-router-1-credential'),
|
||||
preparedTarget,
|
||||
);
|
||||
const deployment = this.serverDeployment('qinglong-workers', {
|
||||
apiVersion: 'apps/v1',
|
||||
kind: 'Deployment',
|
||||
metadata: {
|
||||
name: 'edge-router-1',
|
||||
labels: { 'app.kubernetes.io/component': 'worker' },
|
||||
},
|
||||
spec: {
|
||||
replicas: 1,
|
||||
strategy: { type: 'Recreate' },
|
||||
template: {
|
||||
metadata: {
|
||||
labels: { 'app.kubernetes.io/component': 'worker' },
|
||||
},
|
||||
spec: {
|
||||
containers: [{ name: 'worker', image: 'worker:test' }],
|
||||
volumes: [{
|
||||
name: 'projected-authority',
|
||||
projected: {
|
||||
sources: [{
|
||||
secret: {
|
||||
name: 'edge-router-1-credential',
|
||||
items: [{
|
||||
key: 'credential.token',
|
||||
path: 'credential-token',
|
||||
}],
|
||||
},
|
||||
}],
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
},
|
||||
}, null);
|
||||
this.deployments.set(this.key('qinglong-workers', 'edge-router-1'), deployment);
|
||||
}
|
||||
|
||||
key(namespace, name) {
|
||||
return `${namespace}/${name}`;
|
||||
}
|
||||
|
||||
serverSecret(namespace, body, current) {
|
||||
this.revision += 1;
|
||||
const metadata = body.metadata ?? {};
|
||||
return {
|
||||
...copy(body),
|
||||
metadata: {
|
||||
...copy(metadata),
|
||||
namespace,
|
||||
uid: current?.metadata?.uid ?? `uid-${++this.uid}`,
|
||||
resourceVersion: String(this.revision),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
serverDeployment(namespace, body, current) {
|
||||
this.revision += 1;
|
||||
return {
|
||||
...copy(body),
|
||||
metadata: {
|
||||
...copy(body.metadata),
|
||||
namespace,
|
||||
uid: current?.metadata?.uid ?? `uid-${++this.uid}`,
|
||||
resourceVersion: String(this.revision),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async readNamespacedSecret({ namespace, name }) {
|
||||
const value = this.items.get(this.key(namespace, name));
|
||||
if (!value) throw apiError(404);
|
||||
return copy(value);
|
||||
}
|
||||
|
||||
async createNamespacedSecret({ namespace, body }) {
|
||||
const key = this.key(namespace, body.metadata.name);
|
||||
if (this.items.has(key)) throw apiError(409);
|
||||
const created = this.serverSecret(namespace, body, null);
|
||||
this.items.set(key, created);
|
||||
if (this.failAfterCreate) {
|
||||
this.failAfterCreate = false;
|
||||
throw apiError(409);
|
||||
}
|
||||
return copy(created);
|
||||
}
|
||||
|
||||
async replaceNamespacedSecret({ namespace, name, body }) {
|
||||
const key = this.key(namespace, name);
|
||||
const current = this.items.get(key);
|
||||
if (!current) throw apiError(404);
|
||||
this.replacements.push({
|
||||
expectedResourceVersion: body.metadata.resourceVersion,
|
||||
observedResourceVersion: current.metadata.resourceVersion,
|
||||
});
|
||||
if (body.metadata.resourceVersion !== current.metadata.resourceVersion) {
|
||||
throw apiError(409);
|
||||
}
|
||||
const replaced = this.serverSecret(namespace, body, current);
|
||||
this.items.set(key, replaced);
|
||||
if (this.failAfterReplace) {
|
||||
this.failAfterReplace = false;
|
||||
throw apiError(409);
|
||||
}
|
||||
return copy(replaced);
|
||||
}
|
||||
|
||||
async deleteNamespacedSecret({ namespace, name, body }) {
|
||||
const key = this.key(namespace, name);
|
||||
const current = this.items.get(key);
|
||||
if (!current) throw apiError(404);
|
||||
this.deletions.push(copy(body.preconditions));
|
||||
if (
|
||||
body.preconditions.uid !== current.metadata.uid ||
|
||||
body.preconditions.resourceVersion !== current.metadata.resourceVersion
|
||||
) {
|
||||
throw apiError(409);
|
||||
}
|
||||
this.items.delete(key);
|
||||
if (this.failAfterDelete) {
|
||||
this.failAfterDelete = false;
|
||||
throw apiError(404);
|
||||
}
|
||||
return { status: 'Success' };
|
||||
}
|
||||
|
||||
async listNamespacedSecret({ namespace, limit }) {
|
||||
const values = [...this.items.values()].filter((item) =>
|
||||
item.metadata.namespace === namespace &&
|
||||
item.metadata.labels?.['app.kubernetes.io/managed-by'] === 'qinglong3' &&
|
||||
item.metadata.labels?.['qinglong.io/worker-credential-stage'] === 'v1');
|
||||
return {
|
||||
items: copy(values.slice(0, limit)),
|
||||
metadata: values.length > limit ? { _continue: 'opaque' } : {},
|
||||
};
|
||||
}
|
||||
|
||||
async readNamespacedDeployment({ namespace, name }) {
|
||||
const value = this.deployments.get(this.key(namespace, name));
|
||||
if (!value) throw apiError(404);
|
||||
return copy(value);
|
||||
}
|
||||
|
||||
async replaceNamespacedDeployment({ namespace, name, body }) {
|
||||
const key = this.key(namespace, name);
|
||||
const current = this.deployments.get(key);
|
||||
if (!current) throw apiError(404);
|
||||
this.deploymentReplacements.push({
|
||||
expectedResourceVersion: body.metadata.resourceVersion,
|
||||
observedResourceVersion: current.metadata.resourceVersion,
|
||||
});
|
||||
if (body.metadata.resourceVersion !== current.metadata.resourceVersion) {
|
||||
throw apiError(409);
|
||||
}
|
||||
if (this.failBeforeDeploymentReplace) {
|
||||
this.failBeforeDeploymentReplace = false;
|
||||
throw apiError(503);
|
||||
}
|
||||
const replaced = this.serverDeployment(namespace, body, current);
|
||||
this.deployments.set(key, replaced);
|
||||
if (this.failAfterDeploymentReplace) {
|
||||
this.failAfterDeploymentReplace = false;
|
||||
throw apiError(409);
|
||||
}
|
||||
return copy(replaced);
|
||||
}
|
||||
|
||||
get(namespace, name) {
|
||||
return copy(this.items.get(this.key(namespace, name)));
|
||||
}
|
||||
|
||||
getDeployment(namespace, name) {
|
||||
return copy(this.deployments.get(this.key(namespace, name)));
|
||||
}
|
||||
}
|
||||
|
||||
function adapter(api = new FakeKubernetesSecretApi()) {
|
||||
return {
|
||||
api,
|
||||
adapter: new WorkerCredentialKubernetesDeliveryAdapter(api, api, {
|
||||
clusterIdentity: 'cluster-production-a',
|
||||
namespace: 'qinglong-workers',
|
||||
stageNamespace: 'qinglong-workers-staging',
|
||||
targetSecretName: 'edge-router-1-credential',
|
||||
targetDeploymentName: 'edge-router-1',
|
||||
targetDataKey: 'credential.token',
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function token(credentialId, fill) {
|
||||
return Buffer.from(formatWorkerCredentialToken(
|
||||
credentialId,
|
||||
Buffer.alloc(32, fill).toString('base64url'),
|
||||
));
|
||||
}
|
||||
|
||||
function intent(deliveryAdapter, deliveryId, credentialId, material, overrides = {}) {
|
||||
return {
|
||||
deliveryId,
|
||||
workerId: 'edge-router-1',
|
||||
credentialId,
|
||||
credentialVersion: 1,
|
||||
previousCredentialId: null,
|
||||
secretDigest: 'a'.repeat(64),
|
||||
tokenDigest: workerCredentialDeliveryTokenDigest(material),
|
||||
deploymentTargetDigest: deliveryAdapter.deploymentTargetDigest,
|
||||
deploymentGeneration: `generation-${credentialId}`,
|
||||
stagedAtMs: 1_000,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function committed(candidate) {
|
||||
return {
|
||||
...candidate,
|
||||
version: 1,
|
||||
state: 'credential_committed',
|
||||
credentialCommittedAtMs: candidate.stagedAtMs,
|
||||
publishedAtMs: null,
|
||||
publicationDigest: null,
|
||||
observedAtMs: null,
|
||||
observedSessionId: null,
|
||||
observedSessionVersion: null,
|
||||
previousRevokedAtMs: null,
|
||||
};
|
||||
}
|
||||
|
||||
test('plans the exact deployment target digest without Kubernetes authority', () => {
|
||||
const options = {
|
||||
clusterIdentity: 'cluster-production-a',
|
||||
namespace: 'qinglong-workers',
|
||||
stageNamespace: 'qinglong-workers-staging',
|
||||
targetSecretName: 'edge-router-1-credential',
|
||||
targetDeploymentName: 'edge-router-1',
|
||||
targetDataKey: 'credential.token',
|
||||
};
|
||||
const { adapter: delivery } = adapter();
|
||||
assert.equal(
|
||||
workerCredentialKubernetesDeploymentTargetDigest(options),
|
||||
delivery.deploymentTargetDigest,
|
||||
);
|
||||
assert.throws(
|
||||
() => workerCredentialKubernetesDeploymentTargetDigest({
|
||||
...options,
|
||||
namespace: options.stageNamespace,
|
||||
}),
|
||||
/delivery options are invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
test('creates one immutable stage and lists only bounded normalized intent', async () => {
|
||||
const { api, adapter: delivery } = adapter();
|
||||
const material = token('worker_generation_1', 1);
|
||||
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
|
||||
api.failAfterCreate = true;
|
||||
await delivery.stage(candidate, material);
|
||||
assert.deepEqual(await delivery.inspect(IDS[0]), candidate);
|
||||
assert.deepEqual(await delivery.listStaged({ limit: 1 }), {
|
||||
stages: [candidate],
|
||||
truncated: false,
|
||||
});
|
||||
const stored = api.get(
|
||||
'qinglong-workers-staging',
|
||||
`ql3w-stage-${IDS[0].replaceAll('-', '')}`,
|
||||
);
|
||||
assert.equal(stored.immutable, true);
|
||||
assert.equal(stored.type, 'qinglong.io/worker-credential-stage-v1');
|
||||
assert.deepEqual(Object.keys(stored.data), ['credentialToken']);
|
||||
|
||||
await delivery.stage(candidate, material);
|
||||
await assert.rejects(
|
||||
delivery.stage({ ...candidate, deploymentGeneration: 'drift' }, material),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
material.fill(0);
|
||||
});
|
||||
|
||||
test('publishes with resourceVersion CAS and replays a lost update response', async () => {
|
||||
const { api, adapter: delivery } = adapter();
|
||||
const first = token('worker_generation_1', 1);
|
||||
const firstIntent = intent(delivery, IDS[0], 'worker_generation_1', first);
|
||||
await delivery.stage(firstIntent, first);
|
||||
const initial = await delivery.publish(committed(firstIntent));
|
||||
assert.match(initial.publicationDigest, /^[0-9a-f]{64}$/);
|
||||
|
||||
const second = token('worker_generation_2', 2);
|
||||
const secondIntent = intent(
|
||||
delivery,
|
||||
IDS[1],
|
||||
'worker_generation_2',
|
||||
second,
|
||||
{ previousCredentialId: 'worker_generation_1' },
|
||||
);
|
||||
await delivery.stage(secondIntent, second);
|
||||
api.failAfterReplace = true;
|
||||
api.failAfterDeploymentReplace = true;
|
||||
const rotated = await delivery.publish(committed(secondIntent));
|
||||
assert.match(rotated.publicationDigest, /^[0-9a-f]{64}$/);
|
||||
assert.deepEqual(
|
||||
await delivery.publish(committed(secondIntent)),
|
||||
rotated,
|
||||
);
|
||||
assert.equal(api.replacements.length, 2);
|
||||
assert.equal(
|
||||
api.replacements[0].expectedResourceVersion,
|
||||
api.replacements[0].observedResourceVersion,
|
||||
);
|
||||
assert.equal(api.deploymentReplacements.length, 2);
|
||||
assert.equal(
|
||||
api.deploymentReplacements[1].expectedResourceVersion,
|
||||
api.deploymentReplacements[1].observedResourceVersion,
|
||||
);
|
||||
const target = api.get('qinglong-workers', 'edge-router-1-credential');
|
||||
assert.match(
|
||||
target.metadata.labels['qinglong.io/worker-credential-target-digest'],
|
||||
/^[A-Za-z0-9_-]{43}$/,
|
||||
);
|
||||
assert.deepEqual(
|
||||
Buffer.from(target.data['credential.token'], 'base64'),
|
||||
second,
|
||||
);
|
||||
const deployment = api.getDeployment('qinglong-workers', 'edge-router-1');
|
||||
const annotations = deployment.spec.template.metadata.annotations;
|
||||
assert.equal(
|
||||
annotations['qinglong.io/worker-credential-generation'],
|
||||
secondIntent.deploymentGeneration,
|
||||
);
|
||||
assert.equal(
|
||||
annotations['qinglong.io/worker-credential-id'],
|
||||
secondIntent.credentialId,
|
||||
);
|
||||
assert.equal(
|
||||
annotations['qinglong.io/worker-credential-publication-digest'],
|
||||
rotated.publicationDigest,
|
||||
);
|
||||
first.fill(0);
|
||||
second.fill(0);
|
||||
});
|
||||
|
||||
test('recovers a Secret-first crash before advancing the Recreate PodTemplate', async () => {
|
||||
const { api, adapter: delivery } = adapter();
|
||||
const material = token('worker_generation_1', 1);
|
||||
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
|
||||
await delivery.stage(candidate, material);
|
||||
api.failBeforeDeploymentReplace = true;
|
||||
await assert.rejects(
|
||||
delivery.publish(committed(candidate)),
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
);
|
||||
assert.ok(api.get('qinglong-workers', 'edge-router-1-credential'));
|
||||
assert.equal(
|
||||
api.getDeployment('qinglong-workers', 'edge-router-1')
|
||||
.spec.template.metadata.annotations,
|
||||
undefined,
|
||||
);
|
||||
const recovered = await delivery.publish(committed(candidate));
|
||||
assert.match(recovered.publicationDigest, /^[0-9a-f]{64}$/);
|
||||
assert.equal(api.replacements.length, 1);
|
||||
assert.equal(api.deploymentReplacements.length, 2);
|
||||
material.fill(0);
|
||||
});
|
||||
|
||||
test('rejects Deployment drift before mutating the target Secret', async () => {
|
||||
const { api, adapter: delivery } = adapter();
|
||||
const material = token('worker_generation_1', 1);
|
||||
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
|
||||
await delivery.stage(candidate, material);
|
||||
api.deployments.get('qinglong-workers/edge-router-1').spec.strategy.type =
|
||||
'RollingUpdate';
|
||||
await assert.rejects(
|
||||
delivery.publish(committed(candidate)),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
const prepared = api.get('qinglong-workers', 'edge-router-1-credential');
|
||||
assert.equal(
|
||||
prepared.metadata.labels['qinglong.io/worker-credential-target'],
|
||||
'prepared-v3',
|
||||
);
|
||||
assert.deepEqual(prepared.data, {});
|
||||
material.fill(0);
|
||||
});
|
||||
|
||||
test('gives two concurrent rotations from one resourceVersion exactly one winner', async () => {
|
||||
const { adapter: delivery } = adapter();
|
||||
const first = token('worker_generation_1', 1);
|
||||
const firstIntent = intent(delivery, IDS[0], 'worker_generation_1', first);
|
||||
await delivery.stage(firstIntent, first);
|
||||
await delivery.publish(committed(firstIntent));
|
||||
|
||||
const second = token('worker_generation_2', 2);
|
||||
const third = token('worker_generation_3', 3);
|
||||
const secondIntent = intent(delivery, IDS[1], 'worker_generation_2', second, {
|
||||
previousCredentialId: 'worker_generation_1',
|
||||
});
|
||||
const thirdIntent = intent(delivery, IDS[2], 'worker_generation_3', third, {
|
||||
previousCredentialId: 'worker_generation_1',
|
||||
});
|
||||
await delivery.stage(secondIntent, second);
|
||||
await delivery.stage(thirdIntent, third);
|
||||
const results = await Promise.allSettled([
|
||||
delivery.publish(committed(secondIntent)),
|
||||
delivery.publish(committed(thirdIntent)),
|
||||
]);
|
||||
assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1);
|
||||
const rejection = results.find((result) => result.status === 'rejected');
|
||||
assert.ok(rejection.reason instanceof WorkerCredentialDeliveryConflictError);
|
||||
first.fill(0);
|
||||
second.fill(0);
|
||||
third.fill(0);
|
||||
});
|
||||
|
||||
test('deletes only an exact unpublished stage with UID and resourceVersion fences', async () => {
|
||||
const { api, adapter: delivery } = adapter();
|
||||
const material = token('worker_generation_1', 1);
|
||||
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
|
||||
await delivery.stage(candidate, material);
|
||||
const staged = api.get(
|
||||
'qinglong-workers-staging',
|
||||
`ql3w-stage-${IDS[0].replaceAll('-', '')}`,
|
||||
);
|
||||
api.failAfterDelete = true;
|
||||
await delivery.discard(candidate);
|
||||
assert.equal(await delivery.inspect(IDS[0]), null);
|
||||
assert.deepEqual(api.deletions, [{
|
||||
uid: staged.metadata.uid,
|
||||
resourceVersion: staged.metadata.resourceVersion,
|
||||
}]);
|
||||
await delivery.discard(candidate);
|
||||
material.fill(0);
|
||||
});
|
||||
|
||||
test('never discards a stage whose token is already the published target', async () => {
|
||||
const { adapter: delivery } = adapter();
|
||||
const material = token('worker_generation_1', 1);
|
||||
const candidate = intent(delivery, IDS[0], 'worker_generation_1', material);
|
||||
await delivery.stage(candidate, material);
|
||||
await delivery.publish(committed(candidate));
|
||||
await assert.rejects(
|
||||
delivery.discard(candidate),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
assert.deepEqual(await delivery.inspect(IDS[0]), candidate);
|
||||
material.fill(0);
|
||||
});
|
||||
|
||||
test('fails closed when the Kubernetes stage inventory exceeds its hard cap', async () => {
|
||||
const { api, adapter: delivery } = adapter();
|
||||
for (let index = 0; index <= MAX_WORKER_CREDENTIAL_KUBERNETES_STAGES; index += 1) {
|
||||
api.items.set(`synthetic/${index}`, {
|
||||
metadata: {
|
||||
namespace: 'qinglong-workers-staging',
|
||||
labels: {
|
||||
'app.kubernetes.io/managed-by': 'qinglong3',
|
||||
'qinglong.io/worker-credential-stage': 'v1',
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
await assert.rejects(
|
||||
delivery.listStaged(),
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
);
|
||||
});
|
||||
|
||||
test('fails closed when the prepared target Secret is absent or has drifted', async () => {
|
||||
const missing = new FakeKubernetesSecretApi();
|
||||
missing.items.delete('qinglong-workers/edge-router-1-credential');
|
||||
const { adapter: missingDelivery } = adapter(missing);
|
||||
const missingMaterial = token('worker_generation_1', 1);
|
||||
const missingIntent = intent(
|
||||
missingDelivery,
|
||||
IDS[0],
|
||||
'worker_generation_1',
|
||||
missingMaterial,
|
||||
);
|
||||
await missingDelivery.stage(missingIntent, missingMaterial);
|
||||
await assert.rejects(
|
||||
missingDelivery.publish(committed(missingIntent)),
|
||||
WorkerCredentialDeliveryUnavailableError,
|
||||
);
|
||||
assert.equal(missing.replacements.length, 0);
|
||||
|
||||
const drifted = new FakeKubernetesSecretApi();
|
||||
drifted.items.get('qinglong-workers/edge-router-1-credential')
|
||||
.metadata.annotations['unexpected.example/authority'] = 'drift';
|
||||
const { adapter: driftedDelivery } = adapter(drifted);
|
||||
const driftedMaterial = token('worker_generation_2', 2);
|
||||
const driftedIntent = intent(
|
||||
driftedDelivery,
|
||||
IDS[1],
|
||||
'worker_generation_2',
|
||||
driftedMaterial,
|
||||
);
|
||||
await driftedDelivery.stage(driftedIntent, driftedMaterial);
|
||||
await assert.rejects(
|
||||
driftedDelivery.publish(committed(driftedIntent)),
|
||||
WorkerCredentialDeliveryConflictError,
|
||||
);
|
||||
assert.equal(drifted.replacements.length, 0);
|
||||
|
||||
missingMaterial.fill(0);
|
||||
driftedMaterial.fill(0);
|
||||
});
|
||||
Reference in New Issue
Block a user