Projects
Eulaceura:Factory
ocf
_service:obs_scm:0003-qos-add-load-balance.patch
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:obs_scm:0003-qos-add-load-balance.patch of Package ocf
From 4d7ec05fe2cf796d499328546ba1057d08b315c1 Mon Sep 17 00:00:00 2001 From: Kemeng Shi <shikemeng@huawei.com> Date: Tue, 10 Jan 2023 17:52:04 +0800 Subject: [PATCH 3/4] qos: add load balance add load balance to offload IO from cache device to backdev device when cache device is busy. Signed-off-by: Kemeng Shi <shikemeng@huawei.com> --- inc/xcache_io.h | 22 ++++- src/engine/xcache_engine_common.c | 82 +++++++++++++++-- src/engine/xcache_engine_common.h | 63 +++++++++++++ src/engine/xcache_engine_rd.c | 24 +++++ src/engine/xcache_engine_wb.c | 14 +++ src/engine/xcache_engine_wt.c | 8 ++ src/qos/qos.c | 6 ++ src/qos/qos.h | 27 ++++++ src/qos/qos_lb.h | 143 ++++++++++++++++++++++++++++++ src/xcache.c | 1 + src/xcache.h | 3 + src/xcache_ocf_core.c | 4 + 12 files changed, 386 insertions(+), 11 deletions(-) create mode 100644 src/qos/qos.c create mode 100644 src/qos/qos.h create mode 100644 src/qos/qos_lb.h diff --git a/inc/xcache_io.h b/inc/xcache_io.h index a8acb67..833a73c 100644 --- a/inc/xcache_io.h +++ b/inc/xcache_io.h @@ -2,6 +2,7 @@ #define XCACHE_IO_H__ #include "ocf_env.h" +#include "ocf/ocf.h" enum entry_type { XCACHE_IO_ENTRY, @@ -49,6 +50,11 @@ struct backdev_io_end_arg { struct xcache_backdev_io; typedef int (*backdev_io_end_fn)(struct xcache_backdev_io *io, struct backdev_io_end_arg *arg); typedef void (*backdev_io_res_fn)(struct xcache_backdev_io *io); +enum xcache_dir { + XCACHE_RD = 0, + XCACHE_WR, + XCACHE_FLUSH, +}; struct xcache_backdev_io { // queue_entry enum entry_type type; @@ -60,9 +66,19 @@ struct xcache_backdev_io { ocf_cache_line_t line; ctx_data_t *data; - backdev_io_res_fn io_res; - uint64_t addr; - uint64_t size; + union { + /* for xcache lock */ + struct { + backdev_io_res_fn io_res; + uint64_t addr; + uint64_t size; + }; + /* for io_end callback */ + struct { + int dev; + enum xcache_dir dir; + }; + }; void *priv; }; diff --git a/src/engine/xcache_engine_common.c b/src/engine/xcache_engine_common.c index f1bf022..eb1decb 100644 --- a/src/engine/xcache_engine_common.c +++ b/src/engine/xcache_engine_common.c @@ -4,6 +4,7 @@ #include "../utils/utils_cache_line.h" #include "../metadata/metadata.h" +#include "../xcache.h" #include "xcache_engine_common.h" #include "../xcache_lru.h" #include "../xcache_queue.h" @@ -342,6 +343,55 @@ int xcache_foreach_line(struct xcache_io_context *ctx, xcache_line_handle_func f return 0; } +static int xcache_wr_lb_common_end(struct xcache_backdev_io *backdev_io, struct backdev_io_end_arg *cb_arg) +{ + struct xcache_io *io = backdev_io->xcache_io; + ocf_cache_t cache = xcache_io_cache(io); + ocf_cache_line_t line = backdev_io->line; + uint8_t start_secotr, last_sector; + + xcache_get_sectors_range(cache, cb_arg->addr, cb_arg->size, &start_secotr, &last_sector); + ocf_metadata_start_collision_shared_access(cache, line); + metadata_clear_valid_sec(cache, line, start_secotr, last_sector); + ocf_metadata_end_collision_shared_access(cache, line); + + ocf_io_alock_unlock_wr(ocf_cache_line_concurrency(cache), line); + xcache_queue_free_backdev_io(xcache_io_queue(backdev_io->xcache_io), backdev_io); + return 0; +} + +void xcache_wr_lb_common(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line, + uint64_t addr, uint64_t size) +{ + struct xcache_io *io = backdev_io->xcache_io; + uint64_t start_addr = xcache_io_start_addr(io); + uint64_t buf_offset = addr - start_addr; + + backdev_io->line = line; + backdev_io->end = xcache_wr_lb_common_end; + xcache_backdev_submit_io(backdev_io, false, addr, size, buf_offset, OCF_WRITE); +} + +static inline bool xcache_engine_need_lb(struct xcache_io_handler *handler, struct xcache_line_range *line_range) +{ + struct xcache_io_context *ctx = handler->ctx; + struct xcache_io *io = ctx->io; + ocf_cache_t cache = xcache_io_cache(io); + xcache_context_t *xcache_ctx = xcache_get_ctx(cache); + struct backdev_io *backdev_io = NULL; + + if (handler->lb_fn == NULL) { + return false; + } + + if (handler->need_lb_fn != NULL && + !handler->need_lb_fn(xcache_io_cache(io), line_range)) { + return false; + } + + return xcache_qos_need_lb(&xcache_ctx->qos, xcache_io_dir(io), line_range->size); +} + static int xcache_handle_line(void *priv, uint64_t core_line, uint64_t addr, uint64_t size, uint8_t start_sector, uint8_t last_sector) @@ -350,7 +400,14 @@ static int xcache_handle_line(void *priv, struct xcache_io_context *ctx = handler->ctx; struct xcache_io *io = ctx->io; ocf_cache_t cache = xcache_ctx_cache(ctx); - struct xcache_backdev_io *base_io; + struct xcache_backdev_io *backdev_io; + struct xcache_line_range line_range = { + .addr = addr, + .size = size, + .start_sector = start_sector, + .last_sector = last_sector, + }; + ocf_cache_line_t line; int lock; @@ -368,21 +425,30 @@ static int xcache_handle_line(void *priv, } xcache_submit_miss_line(ctx, core_line, handler->miss_fn); - ctx->hit_no++; - ctx->cache_bytes += size; if (lock == OCF_LOCK_NOT_ACQUIRED) { + ctx->hit_no++; + ctx->cache_bytes += size; return 0; } - base_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io)); - if (base_io == NULL) { + backdev_io = xcache_alloc_backdev_io(io); + if (backdev_io == NULL) { ocf_cache_log(cache, log_err, "alloc base io failed\n"); return -ENOMEM; } - base_io->xcache_io = io; - base_io->data = io->data; - handler->hit_fn(base_io, line, SECTORS_TO_BYTES(start_sector), size, ctx->offset); + + line_range.cache_line = line; + if (xcache_engine_need_lb(handler, &line_range)) + { + ctx->core_bytes += size; + handler->lb_fn(backdev_io, line, addr, size); + return 0; + } + + ctx->hit_no++; + ctx->cache_bytes += size; + handler->hit_fn(backdev_io, line, SECTORS_TO_BYTES(start_sector), size, ctx->offset); return 0; } diff --git a/src/engine/xcache_engine_common.h b/src/engine/xcache_engine_common.h index 3fc168d..fd73519 100644 --- a/src/engine/xcache_engine_common.h +++ b/src/engine/xcache_engine_common.h @@ -3,8 +3,10 @@ #include "../ocf_cache_priv.h" #include "../utils/utils_cache_line.h" +#include "../ocf_def_priv.h" #include "../xcache.h" +#include "../xcache_queue.h" #define INVALID_CORE_LINE ((uint64_t)-1) #define INVALID_LINE ((ocf_cache_line_t)-1) @@ -36,6 +38,13 @@ static inline uint8_t xcache_sector_offset(ocf_cache_t cache, uint64_t sector) return sector & (ocf_line_sectors(cache) - 1); } +static inline void xcache_get_sectors_range(ocf_cache_t cache, uint64_t addr, uint64_t size, uint8_t *start_sector, uint8_t *last_sector) +{ + uint64_t offset = xcache_addr_offset(cache, addr); + *start_sector = BYTES_TO_SECTORS(offset); + *last_sector = BYTES_TO_SECTORS(offset + size - 1); +} + static inline void xcache_io_get_line_range(ocf_cache_t cache, uint64_t addr, uint64_t size, uint64_t *line_first, uint64_t *line_last) { @@ -55,6 +64,15 @@ void xcache_map_cache_line(struct xcache_io_context *ctx, uint64_t cache_line_to_addr(ocf_cache_t cache, ocf_cache_line_t line, uint64_t line_offset); ocf_cache_line_t addr_to_cache_line(ocf_cache_t cache, uint64_t addr); +struct xcache_line_range { + ocf_cache_line_t cache_line; + uint64_t core_line; + uint64_t addr; + uint64_t size; + uint64_t start_sector; + uint64_t last_sector; +}; + typedef int (*xcache_line_handle_func)(void *priv, uint64_t core_line, uint64_t addr, uint64_t size, uint8_t start_sector, uint8_t last_sector); @@ -93,6 +111,11 @@ static inline ocf_queue_t xcache_io_queue(struct xcache_io *io) return io->io_queue; } +static inline int xcache_io_dir(struct xcache_io *io) +{ + return io->rw; +} + static inline ocf_core_t xcache_ctx_core(struct xcache_io_context *ctx) { return xcache_io_core(ctx->io); @@ -108,12 +131,28 @@ static inline ocf_queue_t xcache_ctx_queue(struct xcache_io_context *ctx) return xcache_io_queue(ctx->io); } +static inline struct xcache_backdev_io *xcache_alloc_backdev_io(struct xcache_io *io) +{ + struct xcache_backdev_io *backdev_io = xcache_queue_alloc_backdev_io(xcache_io_queue(io)); + + if (backdev_io == NULL) { + return NULL; + } + + backdev_io->xcache_io = io; + backdev_io->data = io->data; + return backdev_io; +} + typedef int (*xcache_line_valid_fn)(ocf_cache_t cache, ocf_cache_line_t line, uint8_t start_sector, uint8_t last_sector); typedef int (*xcache_line_hit_fn)(struct xcache_backdev_io *base_io, ocf_cache_line_t line, uint64_t offset, uint64_t size, uint64_t buf_offset); typedef int (*xcache_line_miss_fn)(struct xcache_backdev_io *base_io, uint64_t addr, uint64_t size, uint64_t buf_offset); +typedef int (*xcache_line_need_lb_fn)(ocf_cache_t cache, struct xcache_line_range *line_range); +typedef void (*xcache_line_lb_fn)(struct xcache_backdev_io *base_io, ocf_cache_line_t line, + uint64_t addr, uint64_t size); struct xcache_io_handler { struct xcache_io_context *ctx; @@ -121,9 +160,14 @@ struct xcache_io_handler { xcache_line_hit_fn hit_fn; xcache_line_miss_fn miss_fn; backdev_io_res_fn res_fn; + xcache_line_need_lb_fn need_lb_fn; + xcache_line_lb_fn lb_fn; }; int xcache_handle_io(struct xcache_io_handler *handler); +void xcache_wr_lb_common(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line, + uint64_t addr, uint64_t size); + static inline void xcache_io_get(struct xcache_io *io) { env_atomic_inc_return(&io->remaining); @@ -144,9 +188,28 @@ static inline void xcache_io_put(struct xcache_io *io) xcache_io_end(io, io->error); } +static inline xcache_context_t *backdev_io_to_xcache_ctx(struct xcache_backdev_io *io_base) +{ + struct xcache_io *io = io_base->xcache_io; + ocf_queue_t q = io->io_queue; + ocf_cache_t cache = q->cache; + return xcache_get_ctx(cache); +} + static inline void xcache_backdev_submit_io(struct xcache_backdev_io *io_base, bool cached, uint64_t addr, uint64_t size, uint64_t buf_offset, uint8_t dir) { struct xcache_io *io = io_base->xcache_io; + xcache_context_t *xcache_ctx = backdev_io_to_xcache_ctx(io_base); + + io_base->dir = dir; + if (cached) { + io_base->dev = CACHE_DEV; + xcache_qos_load_add(&xcache_ctx->qos, CACHE_DEV, dir, size); + } else { + io_base->dev = CORE_DEV; + xcache_qos_load_add(&xcache_ctx->qos, CORE_DEV, dir, size); + } + xcache_io_get(io); spdk_backdev_submit_io(io_base, cached, addr, size, buf_offset, dir); } diff --git a/src/engine/xcache_engine_rd.c b/src/engine/xcache_engine_rd.c index ffe06d2..6ac3b7e 100644 --- a/src/engine/xcache_engine_rd.c +++ b/src/engine/xcache_engine_rd.c @@ -322,6 +322,28 @@ static int xcache_read_line_valid(ocf_cache_t cache, ocf_cache_line_t line, return !metadata_test_valid_sec(cache, line, start_sector, last_sector); } +static int xcache_read_lb_cb(struct xcache_backdev_io *backdev_io, struct backdev_io_end_arg *cb_arg) +{ + struct xcache_io *io = backdev_io->xcache_io; + ocf_cache_t cache = xcache_io_cache(io); + + ocf_io_alock_unlock_wr(ocf_cache_line_concurrency(cache), backdev_io->line); + xcache_queue_free_backdev_io(xcache_io_queue(backdev_io->xcache_io), backdev_io); + return 0; +} + +static void xcache_read_lb(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line, + uint64_t addr, uint64_t size) +{ + struct xcache_io *io = backdev_io->xcache_io; + uint64_t start_addr = xcache_io_start_addr(io); + uint64_t buf_offset = addr - start_addr; + + backdev_io->end = xcache_read_lb_cb; + backdev_io->line = line; + xcache_backdev_submit_io(backdev_io, false, addr, size, buf_offset, OCF_READ); +} + int xcache_read_generic(struct xcache_io *io) { int ret; @@ -332,6 +354,8 @@ int xcache_read_generic(struct xcache_io *io) .valid_fn = xcache_read_line_valid, .miss_fn = xcache_read_miss, .hit_fn = xcache_read_hit, + .need_lb_fn = NULL, + .lb_fn = xcache_read_lb, }; xcache_init_io_ctx(&ctx, io); diff --git a/src/engine/xcache_engine_wb.c b/src/engine/xcache_engine_wb.c index 064f650..e4f8212 100644 --- a/src/engine/xcache_engine_wb.c +++ b/src/engine/xcache_engine_wb.c @@ -170,6 +170,18 @@ static void xcache_wb_res(struct xcache_backdev_io *base_io) xcache_wb_hit(base_io, base_io->line, offset, base_io->size, buf_offset); } +/* bypass dirty sectors to core will make additional cache io to update valid bit */ +static int xcache_wb_need_lb(ocf_cache_t cache, struct xcache_line_range *range) +{ + return !metadata_test_dirty_sec(cache, range->cache_line, range->start_sector, range->last_sector); +} + +static void xcache_wb_lb(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line, + uint64_t addr, uint64_t size) +{ + xcache_wr_lb_common(backdev_io, line, addr, size); +} + int xcache_wb(struct xcache_io *io) { int ret; @@ -180,6 +192,8 @@ int xcache_wb(struct xcache_io *io) .valid_fn = NULL, .miss_fn = xcache_wb_miss, .hit_fn = xcache_wb_hit, + .need_lb_fn = xcache_wb_need_lb, + .lb_fn = xcache_wb_lb, }; mark_flush(); diff --git a/src/engine/xcache_engine_wt.c b/src/engine/xcache_engine_wt.c index 234608d..e3d4c99 100644 --- a/src/engine/xcache_engine_wt.c +++ b/src/engine/xcache_engine_wt.c @@ -167,6 +167,12 @@ static int xcache_wt_core(struct xcache_io_context *ctx) return 0; } +static void xcache_wt_lb(struct xcache_backdev_io *backdev_io, ocf_cache_line_t line, + uint64_t addr, uint64_t size) +{ + xcache_wr_lb_common(backdev_io, line, addr, size); +} + int xcache_wt(struct xcache_io *io) { struct xcache_io_context ctx; @@ -176,6 +182,8 @@ int xcache_wt(struct xcache_io *io) .valid_fn = NULL, .miss_fn = NULL, .hit_fn = xcache_wt_hit_cache, + .need_lb_fn = NULL, + .lb_fn = xcache_wt_lb, }; int ret; diff --git a/src/qos/qos.c b/src/qos/qos.c new file mode 100644 index 0000000..6ea2da9 --- /dev/null +++ b/src/qos/qos.c @@ -0,0 +1,6 @@ +#include "qos.h" + +void xcache_qos_init(struct xcache_qos *qos) +{ + qos_lb_init(&qos->qos_lb); +} diff --git a/src/qos/qos.h b/src/qos/qos.h new file mode 100644 index 0000000..3b6a691 --- /dev/null +++ b/src/qos/qos.h @@ -0,0 +1,27 @@ +#ifndef __QOS_H__ +#define __QOS_H__ + +#include "qos_lb.h" + +struct xcache_qos { + struct qos_lb qos_lb; +}; + +static inline void xcache_qos_load_add(struct xcache_qos *qos, int dev, enum xcache_dir dir, uint64_t bytes) +{ + qos_lb_load_add(&qos->qos_lb, dev, dir, bytes); +} + +static inline void xcache_qos_load_sub(struct xcache_qos *qos, int dev, enum xcache_dir dir, uint64_t bytes) +{ + qos_lb_load_sub(&qos->qos_lb, dev, dir, bytes); +} + +static inline bool xcache_qos_need_lb(struct xcache_qos *qos, enum xcache_dir dir, uint64_t bytes) +{ + return qos_need_lb(&qos->qos_lb, dir, bytes); +} + +void xcache_qos_init(struct xcache_qos *qos); + +#endif diff --git a/src/qos/qos_lb.h b/src/qos/qos_lb.h new file mode 100644 index 0000000..bb3bfe4 --- /dev/null +++ b/src/qos/qos_lb.h @@ -0,0 +1,143 @@ +#ifndef __QOS_LB_H__ +#define __QOS_LB_H__ + +#define CORE_DEFAULT_LOAD_WEIGHT 30 +#define CACHE_DEFAULT_LOAD_WEIGHT 1 + +#define CORE_DEV 0 +#define CACHE_DEV 1 + +#include <stdint.h> +#include <stdbool.h> + +#include "ocf/xcache.h" + +struct qos_dev_load { + env_atomic64 read_inflight_bytes; + env_atomic64 write_inflight_bytes; + uint32_t read_weight; + uint32_t write_weight; +}; + +static inline void qos_dev_load_init(struct qos_dev_load *load) +{ + env_atomic64_set(&load->read_inflight_bytes, 0); + env_atomic64_set(&load->write_inflight_bytes, 0); +} + +static inline void qos_dev_load_add(struct qos_dev_load *load, enum xcache_dir dir, uint64_t bytes) +{ + switch (dir) { + case XCACHE_RD: + env_atomic64_add(bytes, &load->read_inflight_bytes); + break; + case XCACHE_WR: + env_atomic64_add(bytes, &load->write_inflight_bytes); + break; + default: + break; + } +} + +static inline void qos_dev_load_sub(struct qos_dev_load *load, enum xcache_dir dir, uint64_t bytes) +{ + switch (dir) { + case XCACHE_RD: + env_atomic64_sub(bytes, &load->read_inflight_bytes); + break; + case XCACHE_WR: + env_atomic64_sub(bytes, &load->write_inflight_bytes); + break; + default: + break; + } +} + +static inline uint64_t qos_dev_load_read(struct qos_dev_load *load, enum xcache_dir dir) +{ + switch (dir) { + case XCACHE_RD: + return env_atomic64_read(&load->read_inflight_bytes); + case XCACHE_WR: + return env_atomic64_read(&load->write_inflight_bytes); + default: + return 0; + } +} + +static inline uint32_t qos_dev_load_weight(struct qos_dev_load *load, enum xcache_dir dir) +{ + switch (dir) { + case XCACHE_RD: + return load->read_weight; + case XCACHE_WR: + return load->write_weight; + default: + return 0; + } +} + +static inline uint64_t do_cal_load(uint64_t bytes, uint32_t weight) +{ + return bytes * weight; +} + +static inline uint64_t qos_dev_load_cal(struct qos_dev_load *load) +{ + uint64_t read_inflight_bytes = qos_dev_load_read(load, XCACHE_RD); + uint64_t write_inflight_bytes = qos_dev_load_read(load, XCACHE_WR); + + return do_cal_load(read_inflight_bytes, load->read_weight) + + do_cal_load(write_inflight_bytes, load->write_weight); +} + +struct qos_lb { + struct qos_dev_load cache_load; + struct qos_dev_load core_load; +}; + +static inline void qos_lb_init(struct qos_lb *qos_lb) +{ + qos_dev_load_init(&qos_lb->cache_load); + qos_dev_load_init(&qos_lb->core_load); + qos_lb->cache_load.read_weight = CACHE_DEFAULT_LOAD_WEIGHT; + qos_lb->cache_load.write_weight = CACHE_DEFAULT_LOAD_WEIGHT; + qos_lb->core_load.read_weight = CORE_DEFAULT_LOAD_WEIGHT; + qos_lb->core_load.write_weight = CORE_DEFAULT_LOAD_WEIGHT; +} + +static inline void qos_lb_load_add(struct qos_lb *qos_lb, int dev, enum xcache_dir dir, uint64_t bytes) +{ + switch (dev) { + case CACHE_DEV: + qos_dev_load_add(&qos_lb->cache_load, dir, bytes); + break; + case CORE_DEV: + qos_dev_load_add(&qos_lb->core_load, dir, bytes); + break; + default: + break; + } +} + +static inline void qos_lb_load_sub(struct qos_lb *qos_lb, int dev, enum xcache_dir dir, uint64_t bytes) +{ + switch (dev) { + case CACHE_DEV: + qos_dev_load_sub(&qos_lb->cache_load, dir, bytes); + break; + case CORE_DEV: + qos_dev_load_sub(&qos_lb->core_load, dir, bytes); + break; + default: + break; + } +} + +static inline bool qos_need_lb(struct qos_lb *qos_lb, enum xcache_dir dir, uint64_t bytes) +{ + return qos_dev_load_cal(&qos_lb->cache_load) > qos_dev_load_cal(&qos_lb->core_load) + + do_cal_load(bytes, qos_dev_load_weight(&qos_lb->core_load, dir)); +} + +#endif diff --git a/src/xcache.c b/src/xcache.c index e8d1f2d..0c6a2b8 100644 --- a/src/xcache.c +++ b/src/xcache.c @@ -17,6 +17,7 @@ int xcache_init(ocf_cache_t cache) set_deadline_policy(); evicting_init(ctx); ctx->line_size_shift = __builtin_ffsll(ocf_line_size(cache)) - 1; + xcache_qos_init(&ctx->qos); return 0; } diff --git a/src/xcache.h b/src/xcache.h index f31ec15..4fd7277 100644 --- a/src/xcache.h +++ b/src/xcache.h @@ -5,7 +5,9 @@ #include "ocf/ocf_types.h" #include "./ocf_cache_priv.h" +#include "ocf/xcache.h" #include "xcache_cleaner.h" +#include "qos/qos.h" typedef ocf_cache_line_t xcache_line_t; @@ -14,6 +16,7 @@ typedef struct xcache_context { void *xcache_evicting; ocf_cache_t cache; struct xcache_cleaning_ctx cleaning_ctx; + struct xcache_qos qos; } xcache_context_t; static inline xcache_context_t *xcache_get_ctx(ocf_cache_t cache) diff --git a/src/xcache_ocf_core.c b/src/xcache_ocf_core.c index a3d5c1c..bfd6619 100644 --- a/src/xcache_ocf_core.c +++ b/src/xcache_ocf_core.c @@ -6,6 +6,7 @@ #include "ocf/xcache.h" #include "xcache_queue.h" +#include "qos/qos.h" void xcache_submit_io(struct xcache_io *io) { @@ -34,6 +35,9 @@ void xcache_submit_io(struct xcache_io *io) void xcache_backdev_io_end(struct xcache_backdev_io *bd_io, struct backdev_io_end_arg *arg) { struct xcache_io *io = bd_io->xcache_io; + xcache_context_t *xcache_ctx = backdev_io_to_xcache_ctx(bd_io); + + xcache_qos_load_sub(&xcache_ctx->qos, bd_io->dev, bd_io->dir, arg->size); io->error |= arg->error; bd_io->end(bd_io, arg); -- 2.30.0
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.
浙ICP备2022010568号-2