File _service:tar_scm:0001-iSulad-add-json-files-and... of Package lxc

File _service:tar_scm:0001-iSulad-add-json-files-and-adapt-to-meson.patch of Package lxc (Revision 42c57eb05a334165e18585f1203f9ba5)

Currently displaying revision 42c57eb05a334165e18585f1203f9ba5 , Show latest

From 757bc66c44a58ca2d65eb4c8199ad55cba580d00 Mon Sep 17 00:00:00 2001
From: haozi007 <liuhao27@huawei.com>
Date: Fri, 14 Jul 2023 11:21:56 +0800
Subject: [PATCH 1/3] [iSulad] add json files and adapt to meson

Signed-off-by: haozi007 <liuhao27@huawei.com>
---
 meson.build                      |   30 +-
 meson_options.txt                |    8 +
 src/lxc/af_unix.c                |   50 +
 src/lxc/af_unix.h                |    5 +
 src/lxc/cgroups/isulad_cgfsng.c  | 3229 ++++++++++++++++++++++++++++++
 src/lxc/commands.c               |  179 ++
 src/lxc/commands.h               |   10 +
 src/lxc/commands_utils.c         |   57 +
 src/lxc/commands_utils.h         |    7 +
 src/lxc/exec_commands.c          |  477 +++++
 src/lxc/exec_commands.h          |   77 +
 src/lxc/isulad_utils.c           |  535 +++++
 src/lxc/isulad_utils.h           |  102 +
 src/lxc/json/defs.c              |  205 ++
 src/lxc/json/defs.h              |   37 +
 src/lxc/json/json_common.c       | 1153 +++++++++++
 src/lxc/json/json_common.h       |  194 ++
 src/lxc/json/logger_json_file.c  |  246 +++
 src/lxc/json/logger_json_file.h  |   45 +
 src/lxc/json/oci_runtime_hooks.c |   52 +
 src/lxc/json/oci_runtime_hooks.h |   15 +
 src/lxc/json/oci_runtime_spec.c  |  195 ++
 src/lxc/json/oci_runtime_spec.h  |   37 +
 src/lxc/json/read-file.c         |   95 +
 src/lxc/json/read-file.h         |   11 +
 src/lxc/meson.build              |   26 +-
 src/lxc/path.c                   |  521 +++++
 src/lxc/path.h                   |   33 +
 28 files changed, 7629 insertions(+), 2 deletions(-)
 create mode 100644 src/lxc/cgroups/isulad_cgfsng.c
 create mode 100644 src/lxc/exec_commands.c
 create mode 100644 src/lxc/exec_commands.h
 create mode 100644 src/lxc/isulad_utils.c
 create mode 100644 src/lxc/isulad_utils.h
 create mode 100644 src/lxc/json/defs.c
 create mode 100644 src/lxc/json/defs.h
 create mode 100755 src/lxc/json/json_common.c
 create mode 100755 src/lxc/json/json_common.h
 create mode 100644 src/lxc/json/logger_json_file.c
 create mode 100644 src/lxc/json/logger_json_file.h
 create mode 100644 src/lxc/json/oci_runtime_hooks.c
 create mode 100644 src/lxc/json/oci_runtime_hooks.h
 create mode 100644 src/lxc/json/oci_runtime_spec.c
 create mode 100644 src/lxc/json/oci_runtime_spec.h
 create mode 100644 src/lxc/json/read-file.c
 create mode 100644 src/lxc/json/read-file.h
 create mode 100644 src/lxc/path.c
 create mode 100644 src/lxc/path.h

diff --git a/meson.build b/meson.build
index 1b2d673..fda8045 100644
--- a/meson.build
+++ b/meson.build
@@ -163,6 +163,8 @@ want_seccomp = get_option('seccomp')
 want_thread_safety = get_option('thread-safety')
 want_memfd_rexec = get_option('memfd-rexec')
 want_sd_bus = get_option('sd-bus')
+want_isulad = get_option('isulad')
+want_securecomplie = get_option('securecomplie')
 
 srcconf.set_quoted('DEFAULT_CGROUP_PATTERN', cgrouppattern)
 if coverity
@@ -228,6 +230,19 @@ possible_link_flags = [
     '-fstack-protector-strong',
 ]
 
+if want_isulad
+    possible_cc_flags += ['-D_FORTIFY_SOURCE=2']
+    yajldep = dependency('yajl', version : '>=2')
+    srcconf.set('HAVE_ISULAD', yajldep.found())
+    liblxc_dependencies += yajldep
+else
+    srcconf.set('HAVE_ISULAD', false)
+endif
+
+if want_isulad or want_securecomplie
+    possible_link_flags += ['-Wl,-z,noexecstack']
+endif
+
 # The gold linker fails with a bus error on sparc64
 if build_machine.cpu_family() != 'sparc64'
     possible_link_flags += '-Wl,-fuse-ld=gold'
@@ -881,13 +896,24 @@ template_scripts = []
 test_programs = []
 
 # Includes.
+if want_isulad
 liblxc_includes = include_directories(
     '.',
     'src',
     'src/include',
     'src/lxc',
+    'src/lxc/json',
     'src/lxc/cgroups',
     'src/lxc/storage')
+else
+liblxc_includes = include_directories(
+    '.',
+    'src',
+    'src/include',
+    'src/lxc',
+    'src/lxc/cgroups',
+    'src/lxc/storage')
+endif
 
 # Our static sub-project binaries don't (and in fact can't) link to our
 # dependencies directly, but need access to the headers when compiling (most
@@ -942,7 +968,9 @@ subdir('doc/ja')
 subdir('doc/ko')
 subdir('doc/examples')
 subdir('doc/rootfs')
-subdir('hooks')
+if not want_isulad
+    subdir('hooks')
+endif
 if want_commands
     subdir('src/lxc/cmd')
 endif
diff --git a/meson_options.txt b/meson_options.txt
index 9803473..7146b17 100644
--- a/meson_options.txt
+++ b/meson_options.txt
@@ -120,3 +120,11 @@ option('memfd-rexec', type : 'boolean', value : 'true',
 
 option('distrosysconfdir', type : 'string', value: '',
        description: 'relative path to sysconfdir for distro default configuration')
+
+# was --{disable,enable}-isulad in autotools
+option('isulad', type: 'boolean', value: 'true',
+       description: 'enable adapt to iSulad')
+
+# was --{disable,enable}-securecomplie in autotools
+option('securecomplie', type: 'boolean', value: 'true',
+       description: 'enable secure compile')
\ No newline at end of file
diff --git a/src/lxc/af_unix.c b/src/lxc/af_unix.c
index 6f05b70..6db1864 100644
--- a/src/lxc/af_unix.c
+++ b/src/lxc/af_unix.c
@@ -516,7 +516,11 @@ int lxc_unix_connect_type(struct sockaddr_un *addr, int type)
 	ret = connect(fd, (struct sockaddr *)addr,
 		      offsetof(struct sockaddr_un, sun_path) + len);
 	if (ret < 0)
+#ifdef HAVE_ISULAD
+		return syswarn("Failed to connect AF_UNIX socket");
+#else
 		return syserror("Failed to connect AF_UNIX socket");
+#endif
 
 	return move_fd(fd);
 }
@@ -545,3 +549,49 @@ int lxc_socket_set_timeout(int fd, int rcv_timeout, int snd_timeout)
 
 	return 0;
 }
+
+#ifdef HAVE_ISULAD
+int lxc_named_unix_open(const char *path, int type, int flags)
+{
+	__do_close int fd = -EBADF;
+	int ret;
+	ssize_t len;
+	struct sockaddr_un addr;
+
+	fd = socket(PF_UNIX, type | SOCK_CLOEXEC, 0);
+	if (fd < 0)
+		return -1;
+
+	if (!path)
+		return move_fd(fd);
+
+	len = lxc_unix_sockaddr(&addr, path);
+	if (len < 0)
+		return -1;
+
+	ret = bind(fd, (struct sockaddr *)&addr, len);
+	if (ret < 0)
+		return -1;
+
+	if (chmod(path, 0600) < 0)
+		return -1;
+
+	if (type == SOCK_STREAM) {
+		ret = listen(fd, 100);
+		if (ret < 0)
+			return -1;
+	}
+
+	return move_fd(fd);
+}
+
+int lxc_named_unix_connect(const char *path)
+{
+	struct sockaddr_un addr;
+
+	if (lxc_unix_sockaddr(&addr, path) < 0)
+		return -1;
+
+	return lxc_unix_connect_type(&addr, SOCK_STREAM);
+}
+#endif
diff --git a/src/lxc/af_unix.h b/src/lxc/af_unix.h
index ba119cf..605afc2 100644
--- a/src/lxc/af_unix.h
+++ b/src/lxc/af_unix.h
@@ -168,4 +168,9 @@ static inline void put_unix_fds(struct unix_fds *fds)
 }
 define_cleanup_function(struct unix_fds *, put_unix_fds);
 
+#ifdef HAVE_ISULAD
+__hidden extern int lxc_named_unix_open(const char *path, int type, int flags);
+__hidden extern int lxc_named_unix_connect(const char *path);
+#endif
+
 #endif /* __LXC_AF_UNIX_H */
diff --git a/src/lxc/cgroups/isulad_cgfsng.c b/src/lxc/cgroups/isulad_cgfsng.c
new file mode 100644
index 0000000..dcaa229
--- /dev/null
+++ b/src/lxc/cgroups/isulad_cgfsng.c
@@ -0,0 +1,3229 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+ * Author: lifeng
+ * Create: 2020-11-02
+ * Description: provide container definition
+ * lxc: linux Container library
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+  ******************************************************************************/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <grp.h>
+#include <linux/kdev_t.h>
+#include <linux/types.h>
+#include <poll.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "af_unix.h"
+#include "caps.h"
+#include "cgroup.h"
+#include "cgroup2_devices.h"
+#include "cgroup_utils.h"
+#include "commands.h"
+#include "conf.h"
+#include "config.h"
+#include "log.h"
+#include "macro.h"
+#include "mainloop.h"
+#include "memory_utils.h"
+#include "storage/storage.h"
+#include "utils.h"
+
+#ifndef HAVE_STRLCPY
+#include "include/strlcpy.h"
+#endif
+
+#ifndef HAVE_STRLCAT
+#include "include/strlcat.h"
+#endif
+
+lxc_log_define(isulad_cgfsng, cgroup);
+
+/* Given a pointer to a null-terminated array of pointers, realloc to add one
+ * entry, and point the new entry to NULL. Do not fail. Return the index to the
+ * second-to-last entry - that is, the one which is now available for use
+ * (keeping the list null-terminated).
+ */
+static int append_null_to_list(void ***list)
+{
+	int newentry = 0;
+
+	if (*list)
+		for (; (*list)[newentry]; newentry++)
+			;
+
+	*list = must_realloc(*list, (newentry + 2) * sizeof(void **));
+	(*list)[newentry + 1] = NULL;
+	return newentry;
+}
+
+/* Given a null-terminated array of strings, check whether @entry is one of the
+ * strings.
+ */
+static bool string_in_list(char **list, const char *entry)
+{
+	if (!list)
+		return false;
+
+	for (int i = 0; list[i]; i++)
+		if (strcmp(list[i], entry) == 0)
+			return true;
+
+	return false;
+}
+
+/* Return a copy of @entry prepending "name=", i.e.  turn "systemd" into
+ * "name=systemd". Do not fail.
+ */
+static char *cg_legacy_must_prefix_named(char *entry)
+{
+	size_t len;
+	char *prefixed;
+
+	len = strlen(entry);
+	prefixed = must_realloc(NULL, len + 6);
+
+	memcpy(prefixed, "name=", STRLITERALLEN("name="));
+	memcpy(prefixed + STRLITERALLEN("name="), entry, len);
+	prefixed[len + 5] = '\0';
+
+	return prefixed;
+}
+
+/* Append an entry to the clist. Do not fail. @clist must be NULL the first time
+ * we are called.
+ *
+ * We also handle named subsystems here. Any controller which is not a kernel
+ * subsystem, we prefix "name=". Any which is both a kernel and named subsystem,
+ * we refuse to use because we're not sure which we have here.
+ * (TODO: We could work around this in some cases by just remounting to be
+ * unambiguous, or by comparing mountpoint contents with current cgroup.)
+ *
+ * The last entry will always be NULL.
+ */
+static void must_append_controller(char **klist, char **nlist, char ***clist,
+				   char *entry)
+{
+	int newentry;
+	char *copy;
+
+	if (string_in_list(klist, entry) && string_in_list(nlist, entry)) {
+		ERROR("Refusing to use ambiguous controller \"%s\"", entry);
+		ERROR("It is both a named and kernel subsystem");
+		return;
+	}
+
+	newentry = append_null_to_list((void ***)clist);
+
+	if (strncmp(entry, "name=", 5) == 0)
+		copy = must_copy_string(entry);
+	else if (string_in_list(klist, entry))
+		copy = must_copy_string(entry);
+	else
+		copy = cg_legacy_must_prefix_named(entry);
+
+	(*clist)[newentry] = copy;
+}
+
+/* Given a handler's cgroup data, return the struct hierarchy for the controller
+ * @c, or NULL if there is none.
+ */
+struct hierarchy *get_hierarchy(struct cgroup_ops *ops, const char *controller)
+{
+	if (!ops->hierarchies)
+		return log_trace_errno(NULL, errno, "There are no useable cgroup controllers");
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		if (!controller) {
+			/* This is the empty unified hierarchy. */
+			if (ops->hierarchies[i]->controllers &&
+			    !ops->hierarchies[i]->controllers[0])
+				return ops->hierarchies[i];
+			continue;
+		} else if (pure_unified_layout(ops) &&
+			   strcmp(controller, "devices") == 0) {
+			if (ops->unified->bpf_device_controller)
+				return ops->unified;
+			break;
+		}
+
+		if (string_in_list(ops->hierarchies[i]->controllers, controller))
+			return ops->hierarchies[i];
+	}
+
+	if (controller)
+		WARN("There is no useable %s controller", controller);
+	else
+		WARN("There is no empty unified cgroup hierarchy");
+
+	return ret_set_errno(NULL, ENOENT);
+}
+
+#define BATCH_SIZE 50
+static void batch_realloc(char **mem, size_t oldlen, size_t newlen)
+{
+	int newbatches = (newlen / BATCH_SIZE) + 1;
+	int oldbatches = (oldlen / BATCH_SIZE) + 1;
+
+	if (!*mem || newbatches > oldbatches)
+		*mem = must_realloc(*mem, newbatches * BATCH_SIZE);
+}
+
+static void append_line(char **dest, size_t oldlen, char *new, size_t newlen)
+{
+	size_t full = oldlen + newlen;
+
+	batch_realloc(dest, oldlen, full + 1);
+
+	memcpy(*dest + oldlen, new, newlen + 1);
+}
+
+/* Slurp in a whole file */
+static char *read_file(const char *fnam)
+{
+	__do_free char *buf = NULL, *line = NULL;
+	__do_fclose FILE *f = NULL;
+	size_t len = 0, fulllen = 0;
+	int linelen;
+
+	f = fopen(fnam, "re");
+	if (!f)
+		return NULL;
+
+	while ((linelen = getline(&line, &len, f)) != -1) {
+		append_line(&buf, fulllen, line, linelen);
+		fulllen += linelen;
+	}
+
+	return move_ptr(buf);
+}
+
+static inline bool is_unified_hierarchy(const struct hierarchy *h)
+{
+	return h->version == CGROUP2_SUPER_MAGIC;
+}
+
+/* Given two null-terminated lists of strings, return true if any string is in
+ * both.
+ */
+static bool controller_lists_intersect(char **l1, char **l2)
+{
+	if (!l1 || !l2)
+		return false;
+
+	for (int i = 0; l1[i]; i++)
+		if (string_in_list(l2, l1[i]))
+			return true;
+
+	return false;
+}
+
+/* For a null-terminated list of controllers @clist, return true if any of those
+ * controllers is already listed the null-terminated list of hierarchies @hlist.
+ * Realistically, if one is present, all must be present.
+ */
+static bool controller_list_is_dup(struct hierarchy **hlist, char **clist)
+{
+	if (!hlist)
+		return false;
+
+	for (int i = 0; hlist[i]; i++)
+		if (controller_lists_intersect(hlist[i]->controllers, clist))
+			return true;
+
+	return false;
+}
+
+/* Return true if the controller @entry is found in the null-terminated list of
+ * hierarchies @hlist.
+ */
+static bool controller_found(struct hierarchy **hlist, char *entry)
+{
+	if (!hlist)
+		return false;
+
+	for (int i = 0; hlist[i]; i++)
+		if (string_in_list(hlist[i]->controllers, entry))
+			return true;
+
+	return false;
+}
+
+/* Return true if all of the controllers which we require have been found.  The
+ * required list is  freezer and anything in lxc.cgroup.use.
+ */
+static bool all_controllers_found(struct cgroup_ops *ops)
+{
+	struct hierarchy **hlist;
+
+	if (!ops->cgroup_use)
+		return true;
+
+	hlist = ops->hierarchies;
+	for (char **cur = ops->cgroup_use; cur && *cur; cur++)
+		if (!controller_found(hlist, *cur))
+			return log_error(false, "No %s controller mountpoint found", *cur);
+
+	return true;
+}
+
+/* Get the controllers from a mountinfo line There are other ways we could get
+ * this info. For lxcfs, field 3 is /cgroup/controller-list. For cgroupfs, we
+ * could parse the mount options. But we simply assume that the mountpoint must
+ * be /sys/fs/cgroup/controller-list
+ */
+static char **cg_hybrid_get_controllers(char **klist, char **nlist, char *line,
+					int type)
+{
+	/* The fourth field is /sys/fs/cgroup/comma-delimited-controller-list
+	 * for legacy hierarchies.
+	 */
+	__do_free_string_list char **aret = NULL;
+	int i;
+	char *p2, *tok;
+	char *p = line, *sep = ",";
+
+	for (i = 0; i < 4; i++) {
+		p = strchr(p, ' ');
+		if (!p)
+			return NULL;
+		p++;
+	}
+
+	/* Note, if we change how mountinfo works, then our caller will need to
+	 * verify /sys/fs/cgroup/ in this field.
+	 */
+	if (strncmp(p, DEFAULT_CGROUP_MOUNTPOINT "/", 15) != 0)
+		return log_warn(NULL, "Found hierarchy not under " DEFAULT_CGROUP_MOUNTPOINT ": \"%s\"", p);
+
+	p += 15;
+	p2 = strchr(p, ' ');
+	if (!p2)
+		return log_error(NULL, "Corrupt mountinfo");
+	*p2 = '\0';
+
+	if (type == CGROUP_SUPER_MAGIC) {
+		__do_free char *dup = NULL;
+
+		/* strdup() here for v1 hierarchies. Otherwise
+		 * lxc_iterate_parts() will destroy mountpoints such as
+		 * "/sys/fs/cgroup/cpu,cpuacct".
+		 */
+		dup = must_copy_string(p);
+		if (!dup)
+			return NULL;
+
+		lxc_iterate_parts (tok, dup, sep)
+			must_append_controller(klist, nlist, &aret, tok);
+	}
+	*p2 = ' ';
+
+	return move_ptr(aret);
+}
+
+static char **cg_unified_make_empty_controller(void)
+{
+	__do_free_string_list char **aret = NULL;
+	int newentry;
+
+	newentry = append_null_to_list((void ***)&aret);
+	aret[newentry] = NULL;
+	return move_ptr(aret);
+}
+
+static char **cg_unified_get_controllers(const char *file)
+{
+	__do_free char *buf = NULL;
+	__do_free_string_list char **aret = NULL;
+	char *sep = " \t\n";
+	char *tok;
+
+	buf = read_file(file);
+	if (!buf)
+		return NULL;
+
+	lxc_iterate_parts(tok, buf, sep) {
+		int newentry;
+		char *copy;
+
+		newentry = append_null_to_list((void ***)&aret);
+		copy = must_copy_string(tok);
+		aret[newentry] = copy;
+	}
+
+	return move_ptr(aret);
+}
+
+static struct hierarchy *add_hierarchy(struct hierarchy ***h, char **clist, char *mountpoint,
+				       char *container_base_path, int type)
+{
+	struct hierarchy *new;
+	int newentry;
+
+	new = zalloc(sizeof(*new));
+	new->controllers = clist;
+	new->mountpoint = mountpoint;
+	new->container_base_path = container_base_path;
+	new->version = type;
+	new->cgfd_con = -EBADF;
+	new->cgfd_mon = -EBADF;
+
+	newentry = append_null_to_list((void ***)h);
+	(*h)[newentry] = new;
+	return new;
+}
+
+/* Get a copy of the mountpoint from @line, which is a line from
+ * /proc/self/mountinfo.
+ */
+static char *cg_hybrid_get_mountpoint(char *line)
+{
+	char *p = line, *sret = NULL;
+	size_t len;
+	char *p2;
+
+	for (int i = 0; i < 4; i++) {
+		p = strchr(p, ' ');
+		if (!p)
+			return NULL;
+		p++;
+	}
+
+	if (strncmp(p, DEFAULT_CGROUP_MOUNTPOINT "/", 15) != 0)
+		return NULL;
+
+	p2 = strchr(p + 15, ' ');
+	if (!p2)
+		return NULL;
+	*p2 = '\0';
+
+	len = strlen(p);
+	sret = must_realloc(NULL, len + 1);
+	memcpy(sret, p, len);
+	sret[len] = '\0';
+
+	return sret;
+}
+
+/* Given a multi-line string, return a null-terminated copy of the current line. */
+static char *copy_to_eol(char *p)
+{
+	char *p2, *sret;
+	size_t len;
+
+	p2 = strchr(p, '\n');
+	if (!p2)
+		return NULL;
+
+	len = p2 - p;
+	sret = must_realloc(NULL, len + 1);
+	memcpy(sret, p, len);
+	sret[len] = '\0';
+
+	return sret;
+}
+
+/* cgline: pointer to character after the first ':' in a line in a \n-terminated
+ * /proc/self/cgroup file. Check whether controller c is present.
+ */
+static bool controller_in_clist(char *cgline, char *c)
+{
+	__do_free char *tmp = NULL;
+	char *tok, *eol;
+	size_t len;
+
+	eol = strchr(cgline, ':');
+	if (!eol)
+		return false;
+
+	len = eol - cgline;
+	tmp = must_realloc(NULL, len + 1);
+	memcpy(tmp, cgline, len);
+	tmp[len] = '\0';
+
+	lxc_iterate_parts(tok, tmp, ",")
+		if (strcmp(tok, c) == 0)
+			return true;
+
+	return false;
+}
+
+/* @basecginfo is a copy of /proc/$$/cgroup. Return the current cgroup for
+ * @controller.
+ */
+static char *cg_hybrid_get_current_cgroup(char *basecginfo, char *controller,
+					  int type)
+{
+	char *p = basecginfo;
+
+	for (;;) {
+		bool is_cgv2_base_cgroup = false;
+
+		/* cgroup v2 entry in "/proc/<pid>/cgroup": "0::/some/path" */
+		if ((type == CGROUP2_SUPER_MAGIC) && (*p == '0'))
+			is_cgv2_base_cgroup = true;
+
+		p = strchr(p, ':');
+		if (!p)
+			return NULL;
+		p++;
+
+		if (is_cgv2_base_cgroup || (controller && controller_in_clist(p, controller))) {
+			p = strchr(p, ':');
+			if (!p)
+				return NULL;
+			p++;
+			return copy_to_eol(p);
+		}
+
+		p = strchr(p, '\n');
+		if (!p)
+			return NULL;
+		p++;
+	}
+}
+
+static void must_append_string(char ***list, char *entry)
+{
+	int newentry;
+	char *copy;
+
+	newentry = append_null_to_list((void ***)list);
+	copy = must_copy_string(entry);
+	(*list)[newentry] = copy;
+}
+
+static int get_existing_subsystems(char ***klist, char ***nlist)
+{
+	__do_free char *line = NULL;
+	__do_fclose FILE *f = NULL;
+	size_t len = 0;
+
+	f = fopen("/proc/self/cgroup", "re");
+	if (!f)
+		return -1;
+
+	while (getline(&line, &len, f) != -1) {
+		char *p, *p2, *tok;
+		p = strchr(line, ':');
+		if (!p)
+			continue;
+		p++;
+		p2 = strchr(p, ':');
+		if (!p2)
+			continue;
+		*p2 = '\0';
+
+		/* If the kernel has cgroup v2 support, then /proc/self/cgroup
+		 * contains an entry of the form:
+		 *
+		 *	0::/some/path
+		 *
+		 * In this case we use "cgroup2" as controller name.
+		 */
+		if ((p2 - p) == 0) {
+			must_append_string(klist, "cgroup2");
+			continue;
+		}
+
+		lxc_iterate_parts(tok, p, ",") {
+			if (strncmp(tok, "name=", 5) == 0)
+				must_append_string(nlist, tok);
+			else
+				must_append_string(klist, tok);
+		}
+	}
+
+	return 0;
+}
+
+static char *trim(char *s)
+{
+	size_t len;
+
+	len = strlen(s);
+	while ((len > 1) && (s[len - 1] == '\n'))
+		s[--len] = '\0';
+
+	return s;
+}
+
+static void lxc_cgfsng_print_hierarchies(struct cgroup_ops *ops)
+{
+	int i;
+	struct hierarchy **it;
+
+	if (!ops->hierarchies) {
+		TRACE("  No hierarchies found");
+		return;
+	}
+
+	TRACE("  Hierarchies:");
+	for (i = 0, it = ops->hierarchies; it && *it; it++, i++) {
+		int j;
+		char **cit;
+
+		TRACE("  %d: base_cgroup: %s", i, (*it)->container_base_path ? (*it)->container_base_path : "(null)");
+		TRACE("      mountpoint:  %s", (*it)->mountpoint ? (*it)->mountpoint : "(null)");
+		TRACE("      controllers:");
+		for (j = 0, cit = (*it)->controllers; cit && *cit; cit++, j++)
+			TRACE("      %d: %s", j, *cit);
+	}
+}
+
+static void lxc_cgfsng_print_basecg_debuginfo(char *basecginfo, char **klist,
+					      char **nlist)
+{
+	int k;
+	char **it;
+
+	TRACE("basecginfo is:");
+	TRACE("%s", basecginfo);
+
+	for (k = 0, it = klist; it && *it; it++, k++)
+		TRACE("kernel subsystem %d: %s", k, *it);
+
+	for (k = 0, it = nlist; it && *it; it++, k++)
+		TRACE("named subsystem %d: %s", k, *it);
+}
+
+struct generic_userns_exec_data {
+	struct hierarchy **hierarchies;
+	const char *container_cgroup;
+	struct lxc_conf *conf;
+	uid_t origuid; /* target uid in parent namespace */
+	char *path;
+};
+
+static int isulad_cgroup_tree_remove(struct hierarchy **hierarchies,
+			const char *container_cgroup)
+{
+	if (!container_cgroup || !hierarchies)
+		return 0;
+
+	for (int i = 0; hierarchies[i]; i++) {
+		struct hierarchy *h = hierarchies[i];
+		int ret;
+
+		if (!h->container_full_path) {
+			h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, container_cgroup, NULL);
+		}
+
+		ret = lxc_rm_rf(h->container_full_path);
+		if (ret < 0) {
+			SYSERROR("Failed to destroy \"%s\"", h->container_full_path);
+			return -1;
+		}
+
+		free_disarm(h->container_full_path);
+	}
+
+	return 0;
+}
+
+static int isulad_cgroup_tree_remove_wrapper(void *data)
+{
+	struct generic_userns_exec_data *arg = data;
+	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
+	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
+	int ret;
+
+	if (!lxc_setgroups(0, NULL) && errno != EPERM)
+		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
+
+	ret = setresgid(nsgid, nsgid, nsgid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
+				       (int)nsgid, (int)nsgid, (int)nsgid);
+
+	ret = setresuid(nsuid, nsuid, nsuid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
+				       (int)nsuid, (int)nsuid, (int)nsuid);
+
+	return isulad_cgroup_tree_remove(arg->hierarchies, arg->container_cgroup);
+}
+
+__cgfsng_ops static bool isulad_cgfsng_payload_destroy(struct cgroup_ops *ops,
+						struct lxc_handler *handler)
+{
+	int ret;
+
+	if (!ops) {
+		ERROR("Called with uninitialized cgroup operations");
+		return false;
+	}
+
+	if (ops->no_controller) {
+		DEBUG("no controller found, ignore isulad_cgfsng_payload_destroy");
+		return true;
+	}
+
+	if (!ops->hierarchies) {
+		DEBUG("no hierarchies found, ignore isulad_cgfsng_payload_destroy");
+		return true;
+	}
+
+	if (!handler) {
+		ERROR("Called with uninitialized handler");
+		return false;
+	}
+
+	if (!handler->conf) {
+		ERROR("Called with uninitialized conf");
+		return false;
+	}
+
+#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
+	ret = bpf_program_cgroup_detach(handler->conf->cgroup2_devices);
+	if (ret < 0)
+		WARN("Failed to detach bpf program from cgroup");
+#endif
+
+	if (handler->conf && !lxc_list_empty(&handler->conf->id_map)) {
+		struct generic_userns_exec_data wrap = {
+			.conf			= handler->conf,
+			.container_cgroup	= ops->container_cgroup,
+			.hierarchies		= ops->hierarchies,
+			.origuid		= 0,
+		};
+		ret = userns_exec_1(handler->conf, isulad_cgroup_tree_remove_wrapper,
+				    &wrap, "cgroup_tree_remove_wrapper");
+	} else {
+		ret = isulad_cgroup_tree_remove(ops->hierarchies, ops->container_cgroup);
+	}
+	if (ret < 0) {
+		SYSWARN("Failed to destroy cgroups");
+		return false;
+	}
+
+	return true;
+}
+
+__cgfsng_ops static void isulad_cgfsng_monitor_destroy(struct cgroup_ops *ops,
+						struct lxc_handler *handler)
+{
+	return;
+}
+
+__cgfsng_ops static inline bool isulad_cgfsng_monitor_create(struct cgroup_ops *ops,
+						      struct lxc_handler *handler)
+{
+	return true;
+}
+
+static bool isulad_copy_parent_file(char *path, char *file)
+{
+	int ret;
+	int len = 0;
+	char *value = NULL;
+	char *current = NULL;
+	char *fpath = NULL;
+	char *lastslash = NULL;
+	char oldv;
+
+	fpath = must_make_path(path, file, NULL);
+	current = read_file(fpath);
+
+	if (current == NULL) {
+		SYSERROR("Failed to read file \"%s\"", fpath);
+		free(fpath);
+		return false;
+	}
+
+	if (strcmp(current, "\n") != 0) {
+		free(fpath);
+		free(current);
+		return true;
+	}
+
+	free(fpath);
+	free(current);
+
+	lastslash = strrchr(path, '/');
+	if (lastslash == NULL) {
+		ERROR("Failed to detect \"/\" in \"%s\"", path);
+		return false;
+	}
+	oldv = *lastslash;
+	*lastslash = '\0';
+	fpath = must_make_path(path, file, NULL);
+	*lastslash = oldv;
+	len = lxc_read_from_file(fpath, NULL, 0);
+	if (len <= 0)
+		goto on_error;
+
+	value = must_realloc(NULL, len + 1);
+	ret = lxc_read_from_file(fpath, value, len);
+	if (ret != len)
+		goto on_error;
+	free(fpath);
+
+	fpath = must_make_path(path, file, NULL);
+	ret = lxc_write_to_file(fpath, value, len, false, 0666);
+	if (ret < 0)
+		SYSERROR("Failed to write \"%s\" to file \"%s\"", value, fpath);
+	free(fpath);
+	free(value);
+	return ret >= 0;
+
+on_error:
+	SYSERROR("Failed to read file \"%s\"", fpath);
+	free(fpath);
+	free(value);
+	return false;
+}
+
+static bool build_sub_cpuset_cgroup_dir(char *cgpath)
+{
+	int ret;
+
+	ret = mkdir_p(cgpath, 0755);
+	if (ret < 0) {
+		if (errno != EEXIST) {
+			SYSERROR("Failed to create directory \"%s\"", cgpath);
+			return false;
+		}
+	}
+
+	/* copy parent's settings */
+	if (!isulad_copy_parent_file(cgpath, "cpuset.cpus")) {
+		SYSERROR("Failed to copy \"cpuset.cpus\" settings");
+		return false;
+	}
+
+	/* copy parent's settings */
+	if (!isulad_copy_parent_file(cgpath, "cpuset.mems")) {
+		SYSERROR("Failed to copy \"cpuset.mems\" settings");
+		return false;
+	}
+
+	return true;
+}
+
+static bool isulad_cg_legacy_handle_cpuset_hierarchy(struct hierarchy *h, char *cgname)
+{
+	char *cgpath, *slash;
+	bool sub_mk_success = false;
+
+	if (is_unified_hierarchy(h))
+		return true;
+
+	if (!string_in_list(h->controllers, "cpuset"))
+		return true;
+
+	cgname += strspn(cgname, "/");
+
+	slash = strchr(cgname, '/');
+
+	if (slash != NULL) {
+		while (slash) {
+			*slash = '\0';
+			cgpath = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
+			sub_mk_success = build_sub_cpuset_cgroup_dir(cgpath);
+			free(cgpath);
+			*slash = '/';
+			if (!sub_mk_success) {
+				return false;
+			}
+			slash = strchr(slash + 1, '/');
+		}
+	}
+
+	cgpath = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
+	sub_mk_success = build_sub_cpuset_cgroup_dir(cgpath);
+	free(cgpath);
+	if (!sub_mk_success) {
+		return false;
+	}
+
+	return true;
+}
+
+static int isulad_mkdir_eexist_on_last(const char *dir, mode_t mode)
+{
+	const char *tmp = dir;
+	const char *orig = dir;
+
+	do {
+		int ret;
+		size_t cur_len;
+		char *makeme;
+
+		dir = tmp + strspn(tmp, "/");
+		tmp = dir + strcspn(dir, "/");
+
+		errno = ENOMEM;
+		cur_len = dir - orig;
+		makeme = strndup(orig, cur_len);
+		if (!makeme)
+			return -1;
+
+		ret = mkdir(makeme, mode);
+		if (ret < 0) {
+			if (errno != EEXIST) {
+				SYSERROR("Failed to create directory \"%s\"", makeme);
+				free(makeme);
+				return -1;
+			}
+		}
+		free(makeme);
+
+	} while (tmp != dir);
+
+	return 0;
+}
+
+static bool create_path_for_hierarchy(struct hierarchy *h, char *cgname, int errfd)
+{
+	int ret;
+	__do_free char *path = NULL;
+
+	path = must_make_path(h->mountpoint, h->container_base_path, cgname, NULL);
+
+	if (file_exists(path)) { // it must not already exist
+		ERROR("Cgroup path \"%s\" already exist.", path);
+		lxc_write_error_message(errfd, "%s:%d: Cgroup path \"%s\" already exist.",
+		                        __FILE__, __LINE__, path);
+		return false;
+	}
+
+	if (!isulad_cg_legacy_handle_cpuset_hierarchy(h, cgname)) {
+		ERROR("Failed to handle legacy cpuset controller");
+		return false;
+	}
+
+	ret = isulad_mkdir_eexist_on_last(path, 0755);
+	if (ret < 0) {
+		ERROR("Failed to create cgroup \"%s\"", path);
+		return false;
+	}
+
+	h->cgfd_con = lxc_open_dirfd(path);
+	if (h->cgfd_con < 0)
+		return log_error_errno(false, errno, "Failed to open %s", path);
+
+	if (h->container_full_path == NULL) {
+		h->container_full_path = move_ptr(path);
+	}
+
+	return true;
+}
+
+/* isulad: create hierarchies path, if fail, return the error */
+__cgfsng_ops static inline bool isulad_cgfsng_payload_create(struct cgroup_ops *ops,
+						      struct lxc_handler *handler)
+{
+	int i;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	char *container_cgroup = ops->container_cgroup;
+
+	if (!ops->hierarchies)
+		return true;
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_create");
+		return true;
+	}
+#endif
+
+	if (!container_cgroup) {
+		ERROR("cgfsng_create container_cgroup is invalid");
+		return false;
+	}
+
+	for (i = 0; ops->hierarchies[i]; i++) {
+		if (!create_path_for_hierarchy(ops->hierarchies[i], container_cgroup, ops->errfd)) {
+			SYSERROR("Failed to create %s", ops->hierarchies[i]->container_full_path);
+			return false;
+		}
+	}
+
+	return true;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_monitor_enter(struct cgroup_ops *ops,
+					      struct lxc_handler *handler)
+{
+	return true;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_payload_enter(struct cgroup_ops *ops,
+					      struct lxc_handler *handler)
+{
+	int len;
+	char pidstr[INTTYPE_TO_STRLEN(pid_t)];
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_enter");
+		return true;
+	}
+#endif
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, ENOENT);
+
+	if (!handler || !handler->conf)
+		return ret_set_errno(false, EINVAL);
+
+	len = snprintf(pidstr, sizeof(pidstr), "%d", handler->pid);
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		int ret;
+		char *fullpath;
+		int retry_count = 0;
+		int max_retry = 10;
+
+		fullpath = must_make_path(ops->hierarchies[i]->container_full_path,
+		                          "cgroup.procs", NULL);
+retry:
+		ret = lxc_write_to_file(fullpath, pidstr, len, false, 0666);
+		if (ret != 0) {
+			if (retry_count < max_retry) {
+				SYSERROR("Failed to enter cgroup \"%s\" with retry count:%d", fullpath, retry_count);
+				(void)isulad_cg_legacy_handle_cpuset_hierarchy(ops->hierarchies[i], ops->container_cgroup);
+				(void)isulad_mkdir_eexist_on_last(ops->hierarchies[i]->container_full_path, 0755);
+				usleep(100 * 1000); /* 100 millisecond */
+				retry_count++;
+				goto retry;
+			}
+			SYSERROR("Failed to enter cgroup \"%s\"", fullpath);
+			free(fullpath);
+			return false;
+		}
+		free(fullpath);
+	}
+
+	return true;
+}
+
+static int fchowmodat(int dirfd, const char *path, uid_t chown_uid,
+		      gid_t chown_gid, mode_t chmod_mode)
+{
+	int ret;
+
+	ret = fchownat(dirfd, path, chown_uid, chown_gid,
+		       AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW);
+	if (ret < 0)
+		return log_warn_errno(-1,
+				      errno, "Failed to fchownat(%d, %s, %d, %d, AT_EMPTY_PATH | AT_SYMLINK_NOFOLLOW )",
+				      dirfd, path, (int)chown_uid,
+				      (int)chown_gid);
+
+	ret = fchmodat(dirfd, (*path != '\0') ? path : ".", chmod_mode, 0);
+	if (ret < 0)
+		return log_warn_errno(-1, errno, "Failed to fchmodat(%d, %s, %d, AT_SYMLINK_NOFOLLOW)",
+				      dirfd, path, (int)chmod_mode);
+
+	return 0;
+}
+
+/* chgrp the container cgroups to container group.  We leave
+ * the container owner as cgroup owner.  So we must make the
+ * directories 775 so that the container can create sub-cgroups.
+ *
+ * Also chown the tasks and cgroup.procs files.  Those may not
+ * exist depending on kernel version.
+ */
+static int chown_cgroup_wrapper(void *data)
+{
+	int ret;
+	uid_t destuid;
+	struct generic_userns_exec_data *arg = data;
+	uid_t nsuid = (arg->conf->root_nsuid_map != NULL) ? 0 : arg->conf->init_uid;
+	gid_t nsgid = (arg->conf->root_nsgid_map != NULL) ? 0 : arg->conf->init_gid;
+
+	if (!lxc_setgroups(0, NULL) && errno != EPERM)
+		return log_error_errno(-1, errno, "Failed to setgroups(0, NULL)");
+
+	ret = setresgid(nsgid, nsgid, nsgid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresgid(%d, %d, %d)",
+				       (int)nsgid, (int)nsgid, (int)nsgid);
+
+	ret = setresuid(nsuid, nsuid, nsuid);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to setresuid(%d, %d, %d)",
+				       (int)nsuid, (int)nsuid, (int)nsuid);
+
+	destuid = get_ns_uid(arg->origuid);
+	if (destuid == LXC_INVALID_UID)
+		destuid = 0;
+
+	for (int i = 0; arg->hierarchies[i]; i++) {
+		int dirfd = arg->hierarchies[i]->cgfd_con;
+
+		(void)fchowmodat(dirfd, "", destuid, nsgid, 0775);
+
+		/*
+		 * Failures to chown() these are inconvenient but not
+		 * detrimental We leave these owned by the container launcher,
+		 * so that container root can write to the files to attach.  We
+		 * chmod() them 664 so that container systemd can write to the
+		 * files (which systemd in wily insists on doing).
+		 */
+
+		if (arg->hierarchies[i]->version == CGROUP_SUPER_MAGIC)
+			(void)fchowmodat(dirfd, "tasks", destuid, nsgid, 0664);
+
+		(void)fchowmodat(dirfd, "cgroup.procs", destuid, nsgid, 0664);
+
+		if (arg->hierarchies[i]->version != CGROUP2_SUPER_MAGIC)
+			continue;
+
+		for (char **p = arg->hierarchies[i]->cgroup2_chown; p && *p; p++)
+			(void)fchowmodat(dirfd, *p, destuid, nsgid, 0664);
+	}
+
+	return 0;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_chown(struct cgroup_ops *ops,
+				      struct lxc_conf *conf)
+{
+	struct generic_userns_exec_data wrap;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, ENOENT);
+
+	if (!conf)
+		return ret_set_errno(false, EINVAL);
+
+	if (lxc_list_empty(&conf->id_map))
+		return true;
+
+	wrap.origuid = geteuid();
+	wrap.path = NULL;
+	wrap.hierarchies = ops->hierarchies;
+	wrap.conf = conf;
+
+	if (userns_exec_1(conf, chown_cgroup_wrapper, &wrap, "chown_cgroup_wrapper") < 0)
+		return log_error_errno(false, errno, "Error requesting cgroup chown in new user namespace");
+
+	return true;
+}
+
+__cgfsng_ops void isulad_cgfsng_payload_finalize(struct cgroup_ops *ops)
+{
+	if (!ops)
+		return;
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_finalize");
+		return;
+	}
+#endif
+
+	if (!ops->hierarchies)
+		return;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		struct hierarchy *h = ops->hierarchies[i];
+		/*
+		 * we don't keep the fds for non-unified hierarchies around
+		 * mainly because we don't make use of them anymore after the
+		 * core cgroup setup is done but also because there are quite a
+		 * lot of them.
+		 */
+		if (!is_unified_hierarchy(h))
+			close_prot_errno_disarm(h->cgfd_con);
+	}
+}
+
+/* cgroup-full:* is done, no need to create subdirs */
+static inline bool cg_mount_needs_subdirs(int type)
+{
+	return !(type >= LXC_AUTO_CGROUP_FULL_RO);
+}
+
+/* After $rootfs/sys/fs/container/controller/the/cg/path has been created,
+ * remount controller ro if needed and bindmount the cgroupfs onto
+ * control/the/cg/path.
+ */
+static int cg_legacy_mount_controllers(int type, struct hierarchy *h,
+				       char *controllerpath, char *cgpath,
+				       const char *container_cgroup)
+{
+	__do_free char *sourcepath = NULL;
+	int ret, remount_flags;
+	int flags = MS_BIND;
+
+	if (type == LXC_AUTO_CGROUP_RO || type == LXC_AUTO_CGROUP_MIXED) {
+		ret = mount(controllerpath, controllerpath, "cgroup", MS_BIND, NULL);
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to bind mount \"%s\" onto \"%s\"",
+					       controllerpath, controllerpath);
+
+		remount_flags = add_required_remount_flags(controllerpath,
+							   controllerpath,
+							   flags | MS_REMOUNT);
+		ret = mount(controllerpath, controllerpath, "cgroup",
+			    remount_flags | MS_REMOUNT | MS_BIND | MS_RDONLY,
+			    NULL);
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to remount \"%s\" ro", controllerpath);
+
+		INFO("Remounted %s read-only", controllerpath);
+	}
+
+	sourcepath = must_make_path(h->mountpoint, h->container_base_path,
+				    container_cgroup, NULL);
+	if (type == LXC_AUTO_CGROUP_RO)
+		flags |= MS_RDONLY;
+
+	ret = mount(sourcepath, cgpath, "cgroup", flags, NULL);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to mount \"%s\" onto \"%s\"",
+				       h->controllers[0], cgpath);
+	INFO("Mounted \"%s\" onto \"%s\"", h->controllers[0], cgpath);
+
+	if (flags & MS_RDONLY) {
+		remount_flags = add_required_remount_flags(sourcepath, cgpath,
+							   flags | MS_REMOUNT);
+		ret = mount(sourcepath, cgpath, "cgroup", remount_flags, NULL);
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to remount \"%s\" ro", cgpath);
+		INFO("Remounted %s read-only", cgpath);
+	}
+
+	INFO("Completed second stage cgroup automounts for \"%s\"", cgpath);
+	return 0;
+}
+
+/* __cg_mount_direct
+ *
+ * Mount cgroup hierarchies directly without using bind-mounts. The main
+ * uses-cases are mounting cgroup hierarchies in cgroup namespaces and mounting
+ * cgroups for the LXC_AUTO_CGROUP_FULL option.
+ */
+static int __cg_mount_direct(int type, struct hierarchy *h,
+			     const char *controllerpath)
+{
+	 __do_free char *controllers = NULL;
+	 char *fstype = "cgroup2";
+	 unsigned long flags = 0;
+	 int ret;
+
+	 flags |= MS_NOSUID;
+	 flags |= MS_NOEXEC;
+	 flags |= MS_NODEV;
+	 flags |= MS_RELATIME;
+
+	 if (type == LXC_AUTO_CGROUP_RO || type == LXC_AUTO_CGROUP_FULL_RO)
+		 flags |= MS_RDONLY;
+
+	 if (h->version != CGROUP2_SUPER_MAGIC) {
+		 controllers = lxc_string_join(",", (const char **)h->controllers, false);
+		 if (!controllers)
+			 return -ENOMEM;
+		 fstype = "cgroup";
+	}
+
+	ret = mount("cgroup", controllerpath, fstype, flags, controllers);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to mount \"%s\" with cgroup filesystem type %s",
+				       controllerpath, fstype);
+
+	DEBUG("Mounted \"%s\" with cgroup filesystem type %s", controllerpath, fstype);
+	return 0;
+}
+
+static inline int cg_mount_in_cgroup_namespace(int type, struct hierarchy *h,
+					       const char *controllerpath)
+{
+	return __cg_mount_direct(type, h, controllerpath);
+}
+
+static inline int cg_mount_cgroup_full(int type, struct hierarchy *h,
+				       const char *controllerpath)
+{
+	if (type < LXC_AUTO_CGROUP_FULL_RO || type > LXC_AUTO_CGROUP_FULL_MIXED)
+		return 0;
+
+	return __cg_mount_direct(type, h, controllerpath);
+}
+
+__cgfsng_ops static bool isulad_cgfsng_mount(struct cgroup_ops *ops,
+                                      struct lxc_handler *handler,
+                                      const char *root, int type)
+{
+	int i, ret;
+	char *tmpfspath = NULL;
+	char *systemdpath = NULL;
+	char *unifiedpath = NULL;
+	bool has_cgns = false, retval = false, wants_force_mount = false;
+	char **merged = NULL;
+
+	if ((type & LXC_AUTO_CGROUP_MASK) == 0)
+		return true;
+
+	if (type & LXC_AUTO_CGROUP_FORCE) {
+		type &= ~LXC_AUTO_CGROUP_FORCE;
+		wants_force_mount = true;
+	}
+
+	if (!wants_force_mount) {
+		if (!lxc_list_empty(&handler->conf->keepcaps))
+			wants_force_mount = !in_caplist(CAP_SYS_ADMIN, &handler->conf->keepcaps);
+		else
+			wants_force_mount = in_caplist(CAP_SYS_ADMIN, &handler->conf->caps);
+	}
+
+	has_cgns = cgns_supported();
+	if (has_cgns && !wants_force_mount)
+		return true;
+
+	if (type == LXC_AUTO_CGROUP_NOSPEC)
+		type = LXC_AUTO_CGROUP_MIXED;
+	else if (type == LXC_AUTO_CGROUP_FULL_NOSPEC)
+		type = LXC_AUTO_CGROUP_FULL_MIXED;
+
+	/* Mount tmpfs */
+	tmpfspath = must_make_path(root, "/sys/fs/cgroup", NULL);
+	if (mkdir_p(tmpfspath, 0755) < 0) {
+		ERROR("Failed to create directory: %s", tmpfspath);
+		goto on_error;
+	}
+
+        if (ops->cgroup_layout == CGROUP_LAYOUT_UNIFIED) {
+                if (has_cgns && wants_force_mount) {
+                        /*
+                         * If cgroup namespaces are supported but the container
+                         * will not have CAP_SYS_ADMIN after it has started we
+                         * need to mount the cgroups manually.
+                         */
+                        return cg_mount_in_cgroup_namespace(type, ops->unified, tmpfspath) == 0;
+                }
+
+                return cg_mount_cgroup_full(type, ops->unified, tmpfspath) == 0;
+        }
+
+	ret = safe_mount(NULL, tmpfspath, "tmpfs",
+	                 MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_RELATIME,
+	                 "size=10240k,mode=755", root, handler->conf->lsm_se_mount_context);
+	if (ret < 0)
+		goto on_error;
+
+	for (i = 0; ops->hierarchies[i]; i++) {
+		char *controllerpath = NULL;
+		char *path2 = NULL;
+		struct hierarchy *h = ops->hierarchies[i];
+		char *controller = strrchr(h->mountpoint, '/');
+
+		if (!controller)
+			continue;
+		controller++;
+
+		// isulad: symlink subcgroup
+		if (strchr(controller, ',') != NULL) {
+			int pret;
+			pret = lxc_append_string(&merged, controller);
+			if (pret < 0)
+				goto on_error;
+		}
+
+		controllerpath = must_make_path(tmpfspath, controller, NULL);
+		if (dir_exists(controllerpath)) {
+			free(controllerpath);
+			continue;
+		}
+
+		ret = mkdir(controllerpath, 0755);
+		if (ret < 0) {
+			SYSERROR("Error creating cgroup path: %s", controllerpath);
+			free(controllerpath);
+			goto on_error;
+		}
+
+		if (has_cgns && wants_force_mount) {
+			/* If cgroup namespaces are supported but the container
+			 * will not have CAP_SYS_ADMIN after it has started we
+			 * need to mount the cgroups manually.
+			 */
+			ret = cg_mount_in_cgroup_namespace(type, h, controllerpath);
+			free(controllerpath);
+			if (ret < 0)
+				goto on_error;
+
+			continue;
+		}
+
+		ret = cg_mount_cgroup_full(type, h, controllerpath);
+		if (ret < 0) {
+			free(controllerpath);
+			goto on_error;
+		}
+
+		if (!cg_mount_needs_subdirs(type)) {
+			free(controllerpath);
+			continue;
+		}
+
+		// isulad: ignore ops->container_cgroup so we will not see directory lxc after /sys/fs/cgroup/xxx in container,
+		// isulad: ignore h->container_base_path so we will not see subgroup of /sys/fs/cgroup/xxx/subgroup in container
+		path2 = must_make_path(controllerpath, NULL);
+		ret = mkdir_p(path2, 0755);
+		if (ret < 0) {
+			free(controllerpath);
+			free(path2);
+			goto on_error;
+		}
+
+		ret = cg_legacy_mount_controllers(type, h, controllerpath,
+		                                  path2, ops->container_cgroup);
+		free(controllerpath);
+		free(path2);
+		if (ret < 0)
+			goto on_error;
+	}
+
+	// isulad: symlink subcgroup
+	if (merged) {
+		char **mc = NULL;
+		for (mc = merged; *mc; mc++) {
+			char *token = NULL;
+			char *copy = must_copy_string(*mc);
+			lxc_iterate_parts(token, copy, ",") {
+				int mret;
+				char *link;
+				link = must_make_path(tmpfspath, token, NULL);
+				mret = symlink(*mc, link);
+				if (mret < 0 && errno != EEXIST) {
+					SYSERROR("Failed to create link %s for target %s", link, *mc);
+					free(copy);
+					free(link);
+					goto on_error;
+				}
+				free(link);
+			}
+			free(copy);
+		}
+	}
+
+
+	// isulad: remount /sys/fs/cgroup to readonly
+	if (type == LXC_AUTO_CGROUP_FULL_RO || type == LXC_AUTO_CGROUP_RO) {
+		ret = mount(tmpfspath, tmpfspath, "bind",
+		            MS_NOSUID|MS_NODEV|MS_NOEXEC|MS_RELATIME|MS_RDONLY|MS_BIND|MS_REMOUNT, NULL);
+		if (ret < 0) {
+			SYSERROR("Failed to remount /sys/fs/cgroup.");
+			goto on_error;
+		}
+	}
+
+	// isulad: remount /sys/fs/cgroup/systemd to readwrite for system container
+	if (handler->conf->systemd != NULL && strcmp(handler->conf->systemd, "true") == 0)
+	{
+		unifiedpath = must_make_path(root, "/sys/fs/cgroup/unified", NULL);
+		if (dir_exists(unifiedpath))
+		{
+			ret = umount2(unifiedpath, MNT_DETACH);
+			if (ret < 0)
+			{
+				SYSERROR("Failed to umount /sys/fs/cgroup/unified.");
+				goto on_error;
+			}
+		}
+
+		systemdpath = must_make_path(root, "/sys/fs/cgroup/systemd", NULL);
+		ret = mount(systemdpath, systemdpath, "bind",
+					MS_NOSUID | MS_NODEV | MS_NOEXEC | MS_RELATIME | MS_BIND | MS_REMOUNT, NULL);
+		if (ret < 0)
+		{
+			SYSERROR("Failed to remount /sys/fs/cgroup/systemd.");
+			goto on_error;
+		}
+	}
+
+	retval = true;
+
+on_error:
+	free(tmpfspath);
+	if (systemdpath != NULL)
+	{
+		free(systemdpath);
+	}
+	if (unifiedpath != NULL)
+	{
+		free(unifiedpath);
+	}
+	lxc_free_array((void **)merged, free);
+	return retval;
+}
+
+/* Only root needs to escape to the cgroup of its init. */
+__cgfsng_ops static bool isulad_cgfsng_escape(const struct cgroup_ops *ops,
+				       struct lxc_conf *conf)
+{
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!conf)
+		return ret_set_errno(false, EINVAL);
+
+	if (conf->cgroup_meta.relative || geteuid())
+		return true;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		__do_free char *fullpath = NULL;
+		int ret;
+
+		fullpath =
+		    must_make_path(ops->hierarchies[i]->mountpoint,
+				   ops->hierarchies[i]->container_base_path,
+				   "cgroup.procs", NULL);
+		ret = lxc_write_to_file(fullpath, "0", 2, false, 0666);
+		if (ret != 0)
+			return log_error_errno(false, errno, "Failed to escape to cgroup \"%s\"", fullpath);
+	}
+
+	return true;
+}
+
+__cgfsng_ops static int isulad_cgfsng_num_hierarchies(struct cgroup_ops *ops)
+{
+	int i = 0;
+
+	if (!ops)
+		return ret_set_errno(-1, ENOENT);
+
+	if (!ops->hierarchies)
+		return 0;
+
+	for (; ops->hierarchies[i]; i++)
+		;
+
+	return i;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_get_hierarchies(struct cgroup_ops *ops, int n,
+						char ***out)
+{
+	int i;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return ret_set_errno(false, ENOENT);
+
+	/* sanity check n */
+	for (i = 0; i < n; i++)
+		if (!ops->hierarchies[i])
+			return ret_set_errno(false, ENOENT);
+
+	*out = ops->hierarchies[i]->controllers;
+
+	return true;
+}
+
+static bool cg_legacy_freeze(struct cgroup_ops *ops)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, "freezer");
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	return lxc_write_openat(h->container_full_path, "freezer.state",
+				"FROZEN", STRLITERALLEN("FROZEN"));
+}
+
+static int freezer_cgroup_events_cb(int fd, uint32_t events, void *cbdata,
+				    struct lxc_epoll_descr *descr)
+{
+	__do_close int duped_fd = -EBADF;
+	__do_free char *line = NULL;
+	__do_fclose FILE *f = NULL;
+	int state = PTR_TO_INT(cbdata);
+	size_t len;
+	const char *state_string;
+
+	duped_fd = dup(fd);
+	if (duped_fd < 0)
+		return LXC_MAINLOOP_ERROR;
+
+	if (lseek(duped_fd, 0, SEEK_SET) < (off_t)-1)
+		return LXC_MAINLOOP_ERROR;
+
+	f = fdopen(duped_fd, "re");
+	if (!f)
+		return LXC_MAINLOOP_ERROR;
+	move_fd(duped_fd);
+
+	if (state == 1)
+		state_string = "frozen 1";
+	else
+		state_string = "frozen 0";
+
+	while (getline(&line, &len, f) != -1)
+		if (strncmp(line, state_string, STRLITERALLEN("frozen") + 2) == 0)
+			return LXC_MAINLOOP_CLOSE;
+
+	return LXC_MAINLOOP_CONTINUE;
+}
+
+static int cg_unified_freeze(struct cgroup_ops *ops, int timeout)
+{
+	__do_close int fd = -EBADF;
+	call_cleaner(lxc_mainloop_close) struct lxc_epoll_descr *descr_ptr = NULL;
+	int ret;
+	struct lxc_epoll_descr descr;
+	struct hierarchy *h;
+
+	h = ops->unified;
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	if (!h->container_full_path)
+		return ret_set_errno(-1, EEXIST);
+
+	if (timeout != 0) {
+		__do_free char *events_file = NULL;
+
+		events_file = must_make_path(h->container_full_path, "cgroup.events", NULL);
+		fd = open(events_file, O_RDONLY | O_CLOEXEC);
+		if (fd < 0)
+			return log_error_errno(-1, errno, "Failed to open cgroup.events file");
+
+		ret = lxc_mainloop_open(&descr);
+		if (ret)
+			return log_error_errno(-1, errno, "Failed to create epoll instance to wait for container freeze");
+
+		/* automatically cleaned up now */
+		descr_ptr = &descr;
+
+		ret = lxc_mainloop_add_handler(&descr, fd, freezer_cgroup_events_cb, INT_TO_PTR((int){1}));
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to add cgroup.events fd handler to mainloop");
+	}
+
+	ret = lxc_write_openat(h->container_full_path, "cgroup.freeze", "1", 1);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to open cgroup.freeze file");
+
+	if (timeout != 0 && lxc_mainloop(&descr, timeout))
+		return log_error_errno(-1, errno, "Failed to wait for container to be frozen");
+
+	return 0;
+}
+
+__cgfsng_ops static int isulad_cgfsng_freeze(struct cgroup_ops *ops, int timeout)
+{
+	if (!ops->hierarchies)
+		return ret_set_errno(-1, ENOENT);
+
+	if (ops->cgroup_layout != CGROUP_LAYOUT_UNIFIED)
+		return cg_legacy_freeze(ops);
+
+	return cg_unified_freeze(ops, timeout);
+}
+
+static int cg_legacy_unfreeze(struct cgroup_ops *ops)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, "freezer");
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	return lxc_write_openat(h->container_full_path, "freezer.state",
+				"THAWED", STRLITERALLEN("THAWED"));
+}
+
+static int cg_unified_unfreeze(struct cgroup_ops *ops, int timeout)
+{
+	__do_close int fd = -EBADF;
+	call_cleaner(lxc_mainloop_close)struct lxc_epoll_descr *descr_ptr = NULL;
+	int ret;
+	struct lxc_epoll_descr descr;
+	struct hierarchy *h;
+
+	h = ops->unified;
+	if (!h)
+		return ret_set_errno(-1, ENOENT);
+
+	if (!h->container_full_path)
+		return ret_set_errno(-1, EEXIST);
+
+	if (timeout != 0) {
+		__do_free char *events_file = NULL;
+
+		events_file = must_make_path(h->container_full_path, "cgroup.events", NULL);
+		fd = open(events_file, O_RDONLY | O_CLOEXEC);
+		if (fd < 0)
+			return log_error_errno(-1, errno, "Failed to open cgroup.events file");
+
+		ret = lxc_mainloop_open(&descr);
+		if (ret)
+			return log_error_errno(-1, errno, "Failed to create epoll instance to wait for container unfreeze");
+
+		/* automatically cleaned up now */
+		descr_ptr = &descr;
+
+		ret = lxc_mainloop_add_handler(&descr, fd, freezer_cgroup_events_cb, INT_TO_PTR((int){0}));
+		if (ret < 0)
+			return log_error_errno(-1, errno, "Failed to add cgroup.events fd handler to mainloop");
+	}
+
+	ret = lxc_write_openat(h->container_full_path, "cgroup.freeze", "0", 1);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to open cgroup.freeze file");
+
+	if (timeout != 0 && lxc_mainloop(&descr, timeout))
+		return log_error_errno(-1, errno, "Failed to wait for container to be unfrozen");
+
+	return 0;
+}
+
+__cgfsng_ops static int isulad_cgfsng_unfreeze(struct cgroup_ops *ops, int timeout)
+{
+	if (!ops->hierarchies)
+		return ret_set_errno(-1, ENOENT);
+
+	if (ops->cgroup_layout != CGROUP_LAYOUT_UNIFIED)
+		return cg_legacy_unfreeze(ops);
+
+	return cg_unified_unfreeze(ops, timeout);
+}
+
+__cgfsng_ops static const char *isulad_cgfsng_get_cgroup(struct cgroup_ops *ops,
+						  const char *controller)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, controller);
+	if (!h)
+		return log_warn_errno(NULL, ENOENT, "Failed to find hierarchy for controller \"%s\"",
+				      controller ? controller : "(null)");
+
+	if (!h->container_full_path)
+		h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, ops->container_cgroup, NULL);
+
+	return h->container_full_path
+		   ? h->container_full_path + strlen(h->mountpoint)
+		   : NULL;
+}
+
+__cgfsng_ops static const char *isulad_cgfsng_get_cgroup_full_path(struct cgroup_ops *ops,
+						  const char *controller)
+{
+	struct hierarchy *h;
+
+	h = get_hierarchy(ops, controller);
+	if (!h)
+		return log_warn_errno(NULL, ENOENT, "Failed to find hierarchy for controller \"%s\"",
+				      controller ? controller : "(null)");
+
+	if (!h->container_full_path)
+		h->container_full_path = must_make_path(h->mountpoint, h->container_base_path, ops->container_cgroup, NULL);
+
+	return h->container_full_path;
+}
+
+/* Given a cgroup path returned from lxc_cmd_get_cgroup_path, build a full path,
+ * which must be freed by the caller.
+ */
+static inline char *build_full_cgpath_from_monitorpath(struct hierarchy *h,
+						       const char *inpath,
+						       const char *filename)
+{
+	return must_make_path(h->mountpoint, inpath, filename, NULL);
+}
+
+static int cgroup_attach_leaf(const struct lxc_conf *conf, int unified_fd, pid_t pid)
+{
+	int idx = 1;
+	int ret;
+	char pidstr[INTTYPE_TO_STRLEN(int64_t) + 1];
+	size_t pidstr_len;
+
+	/* Create leaf cgroup. */
+	ret = mkdirat(unified_fd, ".lxc", 0755);
+	if (ret < 0 && errno != EEXIST)
+		return log_error_errno(-1, errno, "Failed to create leaf cgroup \".lxc\"");
+
+	pidstr_len = sprintf(pidstr, INT64_FMT, (int64_t)pid);
+	ret = lxc_writeat(unified_fd, ".lxc/cgroup.procs", pidstr, pidstr_len);
+	if (ret < 0)
+		ret = lxc_writeat(unified_fd, "cgroup.procs", pidstr, pidstr_len);
+	if (ret == 0)
+		return 0;
+
+	/* this is a non-leaf node */
+	if (errno != EBUSY)
+		return log_error_errno(-1, errno, "Failed to attach to unified cgroup");
+
+	do {
+		bool rm = false;
+		char attach_cgroup[STRLITERALLEN(".lxc-/cgroup.procs") + INTTYPE_TO_STRLEN(int) + 1];
+		char *slash;
+
+		ret = snprintf(attach_cgroup, sizeof(attach_cgroup), ".lxc-%d/cgroup.procs", idx);
+		if (ret < 0 || (size_t)ret >= sizeof(attach_cgroup))
+			return ret_errno(EIO);
+
+		/*
+		 * This shouldn't really happen but the compiler might complain
+		 * that a short write would cause a buffer overrun. So be on
+		 * the safe side.
+		 */
+		if (ret < STRLITERALLEN(".lxc-/cgroup.procs"))
+			return log_error_errno(-EINVAL, EINVAL, "Unexpected short write would cause buffer-overrun");
+
+		slash = &attach_cgroup[ret] - STRLITERALLEN("/cgroup.procs");
+		*slash = '\0';
+
+		ret = mkdirat(unified_fd, attach_cgroup, 0755);
+		if (ret < 0 && errno != EEXIST)
+			return log_error_errno(-1, errno, "Failed to create cgroup %s", attach_cgroup);
+		if (ret == 0)
+			rm = true;
+
+		*slash = '/';
+
+		ret = lxc_writeat(unified_fd, attach_cgroup, pidstr, pidstr_len);
+		if (ret == 0)
+			return 0;
+
+		if (rm && unlinkat(unified_fd, attach_cgroup, AT_REMOVEDIR))
+			SYSERROR("Failed to remove cgroup \"%d(%s)\"", unified_fd, attach_cgroup);
+
+		/* this is a non-leaf node */
+		if (errno != EBUSY)
+			return log_error_errno(-1, errno, "Failed to attach to unified cgroup");
+
+		idx++;
+	} while (idx < 1000);
+
+	return log_error_errno(-1, errno, "Failed to attach to unified cgroup");
+}
+
+static int cgroup_attach_create_leaf(const struct lxc_conf *conf,
+				     int unified_fd, int *sk_fd)
+{
+	__do_close int sk = *sk_fd, target_fd0 = -EBADF, target_fd1 = -EBADF;
+	int target_fds[2];
+	ssize_t ret;
+
+	/* Create leaf cgroup. */
+	ret = mkdirat(unified_fd, ".lxc", 0755);
+	if (ret < 0 && errno != EEXIST)
+		return log_error_errno(-1, errno, "Failed to create leaf cgroup \".lxc\"");
+
+	target_fd0 = openat(unified_fd, ".lxc/cgroup.procs", O_WRONLY | O_CLOEXEC | O_NOFOLLOW);
+	if (target_fd0 < 0)
+		return log_error_errno(-errno, errno, "Failed to open \".lxc/cgroup.procs\"");
+	target_fds[0] = target_fd0;
+
+	target_fd1 = openat(unified_fd, "cgroup.procs", O_WRONLY | O_CLOEXEC | O_NOFOLLOW);
+	if (target_fd1 < 0)
+		return log_error_errno(-errno, errno, "Failed to open \".lxc/cgroup.procs\"");
+	target_fds[1] = target_fd1;
+
+	ret = lxc_abstract_unix_send_fds(sk, target_fds, 2, NULL, 0);
+	if (ret <= 0)
+		return log_error_errno(-errno, errno, "Failed to send \".lxc/cgroup.procs\" fds %d and %d",
+				       target_fd0, target_fd1);
+
+	return log_debug(0, "Sent target cgroup fds %d and %d", target_fd0, target_fd1);
+}
+
+static int cgroup_attach_move_into_leaf(const struct lxc_conf *conf,
+					int *sk_fd, pid_t pid)
+{
+	__do_close int sk = *sk_fd, target_fd0 = -EBADF, target_fd1 = -EBADF;
+	int target_fds[2];
+	char pidstr[INTTYPE_TO_STRLEN(int64_t) + 1];
+	size_t pidstr_len;
+	ssize_t ret;
+
+	ret = lxc_abstract_unix_recv_fds(sk, target_fds, 2, NULL, 0);
+	if (ret <= 0)
+		return log_error_errno(-1, errno, "Failed to receive target cgroup fd");
+	target_fd0 = target_fds[0];
+	target_fd1 = target_fds[1];
+
+	pidstr_len = sprintf(pidstr, INT64_FMT, (int64_t)pid);
+
+	ret = lxc_write_nointr(target_fd0, pidstr, pidstr_len);
+	if (ret > 0 && ret == pidstr_len)
+		return log_debug(0, "Moved process into target cgroup via fd %d", target_fd0);
+
+	ret = lxc_write_nointr(target_fd1, pidstr, pidstr_len);
+	if (ret > 0 && ret == pidstr_len)
+		return log_debug(0, "Moved process into target cgroup via fd %d", target_fd1);
+
+	return log_debug_errno(-1, errno, "Failed to move process into target cgroup via fd %d and %d",
+			       target_fd0, target_fd1);
+}
+
+struct userns_exec_unified_attach_data {
+	const struct lxc_conf *conf;
+	int unified_fd;
+	int sk_pair[2];
+	pid_t pid;
+};
+
+static int cgroup_unified_attach_child_wrapper(void *data)
+{
+	struct userns_exec_unified_attach_data *args = data;
+
+	if (!args->conf || args->unified_fd < 0 || args->pid <= 0 ||
+	    args->sk_pair[0] < 0 || args->sk_pair[1] < 0)
+		return ret_errno(EINVAL);
+
+	close_prot_errno_disarm(args->sk_pair[0]);
+	return cgroup_attach_create_leaf(args->conf, args->unified_fd,
+					 &args->sk_pair[1]);
+}
+
+static int cgroup_unified_attach_parent_wrapper(void *data)
+{
+	struct userns_exec_unified_attach_data *args = data;
+
+	if (!args->conf || args->unified_fd < 0 || args->pid <= 0 ||
+	    args->sk_pair[0] < 0 || args->sk_pair[1] < 0)
+		return ret_errno(EINVAL);
+
+	close_prot_errno_disarm(args->sk_pair[1]);
+	return cgroup_attach_move_into_leaf(args->conf, &args->sk_pair[0],
+					    args->pid);
+}
+
+int cgroup_attach(const struct lxc_conf *conf, const char *name,
+		  const char *lxcpath, pid_t pid)
+{
+	__do_close int unified_fd = -EBADF;
+	int ret;
+
+	if (!conf || !name || !lxcpath || pid <= 0)
+		return ret_errno(EINVAL);
+
+	unified_fd = lxc_cmd_get_cgroup2_fd(name, lxcpath);
+	if (unified_fd < 0)
+		return ret_errno(EBADF);
+
+	if (!lxc_list_empty(&conf->id_map)) {
+		struct userns_exec_unified_attach_data args = {
+			.conf		= conf,
+			.unified_fd	= unified_fd,
+			.pid		= pid,
+		};
+
+		ret = socketpair(PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, args.sk_pair);
+		if (ret < 0)
+			return -errno;
+
+		ret = userns_exec_minimal(conf,
+					  cgroup_unified_attach_parent_wrapper,
+					  &args,
+					  cgroup_unified_attach_child_wrapper,
+					  &args);
+	} else {
+		ret = cgroup_attach_leaf(conf, unified_fd, pid);
+	}
+
+	return ret;
+}
+
+/* Technically, we're always at a delegation boundary here (This is especially
+ * true when cgroup namespaces are available.). The reasoning is that in order
+ * for us to have been able to start a container in the first place the root
+ * cgroup must have been a leaf node. Now, either the container's init system
+ * has populated the cgroup and kept it as a leaf node or it has created
+ * subtrees. In the former case we will simply attach to the leaf node we
+ * created when we started the container in the latter case we create our own
+ * cgroup for the attaching process.
+ */
+static int __cg_unified_attach(const struct hierarchy *h,
+			       const struct lxc_conf *conf, const char *name,
+			       const char *lxcpath, pid_t pid,
+			       const char *controller)
+{
+	__do_close int unified_fd = -EBADF;
+	__do_free char *path = NULL, *cgroup = NULL;
+	int ret;
+
+	if (!conf || !name || !lxcpath || pid <= 0)
+		return ret_errno(EINVAL);
+
+	ret = cgroup_attach(conf, name, lxcpath, pid);
+	if (ret == 0)
+		return log_trace(0, "Attached to unified cgroup via command handler");
+	if (ret != -EBADF)
+		return log_error_errno(ret, errno, "Failed to attach to unified cgroup");
+
+	/* Fall back to retrieving the path for the unified cgroup. */
+	cgroup = lxc_cmd_get_cgroup_path(name, lxcpath, controller);
+	/* not running */
+	if (!cgroup)
+		return 0;
+
+	path = must_make_path(h->mountpoint, cgroup, NULL);
+
+	unified_fd = open(path, O_PATH | O_DIRECTORY | O_CLOEXEC);
+	if (unified_fd < 0)
+		return ret_errno(EBADF);
+
+	if (!lxc_list_empty(&conf->id_map)) {
+		struct userns_exec_unified_attach_data args = {
+			.conf		= conf,
+			.unified_fd	= unified_fd,
+			.pid		= pid,
+		};
+
+		ret = socketpair(PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0, args.sk_pair);
+		if (ret < 0)
+			return -errno;
+
+		ret = userns_exec_minimal(conf,
+					  cgroup_unified_attach_parent_wrapper,
+					  &args,
+					  cgroup_unified_attach_child_wrapper,
+					  &args);
+	} else {
+		ret = cgroup_attach_leaf(conf, unified_fd, pid);
+	}
+
+	return ret;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_attach(struct cgroup_ops *ops,
+				       const struct lxc_conf *conf,
+				       const char *name, const char *lxcpath,
+				       pid_t pid)
+{
+	int len, ret;
+	char pidstr[INTTYPE_TO_STRLEN(pid_t)];
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_attach");
+		return true;
+	}
+#endif
+
+	if (!ops->hierarchies)
+		return true;
+
+	len = snprintf(pidstr, sizeof(pidstr), "%d", pid);
+	if (len < 0 || (size_t)len >= sizeof(pidstr))
+		return false;
+
+	for (int i = 0; ops->hierarchies[i]; i++) {
+		__do_free char *fullpath = NULL, *path = NULL;
+		struct hierarchy *h = ops->hierarchies[i];
+
+		if (h->version == CGROUP2_SUPER_MAGIC) {
+			ret = __cg_unified_attach(h, conf, name, lxcpath, pid,
+						  h->controllers[0]);
+			if (ret < 0)
+				return false;
+
+			continue;
+		}
+
+		path = lxc_cmd_get_cgroup_path(name, lxcpath, h->controllers[0]);
+		/* not running */
+		if (!path)
+			return false;
+
+		fullpath = build_full_cgpath_from_monitorpath(h, path, "cgroup.procs");
+		ret = lxc_write_to_file(fullpath, pidstr, len, false, 0666);
+		if (ret < 0)
+			return log_error_errno(false, errno, "Failed to attach %d to %s",
+					       (int)pid, fullpath);
+	}
+
+	return true;
+}
+
+__cgfsng_ops static int isulad_cgfsng_get(struct cgroup_ops *ops, const char *filename,
+                                   char *value, size_t len, const char *name,
+                                   const char *lxcpath)
+{
+	int ret = -1;
+	size_t controller_len;
+	char *controller, *p, *path;
+	struct hierarchy *h;
+
+	controller_len = strlen(filename);
+	controller = alloca(controller_len + 1);
+	(void)strlcpy(controller, filename, controller_len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+	const char *ori_path = ops->get_cgroup(ops, controller);
+	if (ori_path == NULL) {
+		ERROR("Failed to get cgroup path:%s", controller);
+		return -1;
+	}
+	path = safe_strdup(ori_path);
+
+	h = get_hierarchy(ops, controller);
+	if (h) {
+		char *fullpath;
+
+		fullpath = build_full_cgpath_from_monitorpath(h, path, filename);
+		ret = lxc_read_from_file(fullpath, value, len);
+		free(fullpath);
+	}
+	free(path);
+
+	return ret;
+}
+
+static int device_cgroup_parse_access(struct device_item *device, const char *val)
+{
+	for (int count = 0; count < 3; count++, val++) {
+		switch (*val) {
+		case 'r':
+			device->access[count] = *val;
+			break;
+		case 'w':
+			device->access[count] = *val;
+			break;
+		case 'm':
+			device->access[count] = *val;
+			break;
+		case '\n':
+		case '\0':
+			count = 3;
+			break;
+		default:
+			return ret_errno(EINVAL);
+		}
+	}
+
+	return 0;
+}
+
+int device_cgroup_rule_parse(struct device_item *device, const char *key,
+				    const char *val)
+{
+	int count, ret;
+	char temp[50];
+
+	if (strcmp("devices.allow", key) == 0)
+		device->allow = 1;
+	else
+		device->allow = 0;
+
+	if (strcmp(val, "a") == 0) {
+		/* global rule */
+		device->type = 'a';
+		device->major = -1;
+		device->minor = -1;
+		device->global_rule = device->allow
+					  ? LXC_BPF_DEVICE_CGROUP_BLACKLIST
+					  : LXC_BPF_DEVICE_CGROUP_WHITELIST;
+		device->allow = -1;
+		return 0;
+	}
+
+	/* local rule */
+	device->global_rule = LXC_BPF_DEVICE_CGROUP_LOCAL_RULE;
+
+	switch (*val) {
+	case 'a':
+		__fallthrough;
+	case 'b':
+		__fallthrough;
+	case 'c':
+		device->type = *val;
+		break;
+	default:
+		return -1;
+	}
+
+	val++;
+	if (!isspace(*val))
+		return -1;
+	val++;
+	if (*val == '*') {
+		device->major = -1;
+		val++;
+	} else if (isdigit(*val)) {
+		memset(temp, 0, sizeof(temp));
+		for (count = 0; count < sizeof(temp) - 1; count++) {
+			temp[count] = *val;
+			val++;
+			if (!isdigit(*val))
+				break;
+		}
+		ret = lxc_safe_int(temp, &device->major);
+		if (ret)
+			return -1;
+	} else {
+		return -1;
+	}
+	if (*val != ':')
+		return -1;
+	val++;
+
+	/* read minor */
+	if (*val == '*') {
+		device->minor = -1;
+		val++;
+	} else if (isdigit(*val)) {
+		memset(temp, 0, sizeof(temp));
+		for (count = 0; count < sizeof(temp) - 1; count++) {
+			temp[count] = *val;
+			val++;
+			if (!isdigit(*val))
+				break;
+		}
+		ret = lxc_safe_int(temp, &device->minor);
+		if (ret)
+			return -1;
+	} else {
+		return -1;
+	}
+	if (!isspace(*val))
+		return -1;
+
+	return device_cgroup_parse_access(device, ++val);
+}
+
+__cgfsng_ops static int isulad_cgfsng_set(struct cgroup_ops *ops,
+                                   const char *filename, const char *value,
+                                   const char *name, const char *lxcpath)
+{
+	int ret = -1;
+	size_t controller_len;
+	char *controller, *p, *path;
+	struct hierarchy *h;
+
+	controller_len = strlen(filename);
+	controller = alloca(controller_len + 1);
+	(void)strlcpy(controller, filename, controller_len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+	const char *ori_path = ops->get_cgroup(ops, controller);
+	if (ori_path == NULL) {
+		ERROR("Failed to get cgroup path:%s", controller);
+		return -1;
+	}
+	path = safe_strdup(ori_path);
+
+	h = get_hierarchy(ops, controller);
+	if (h) {
+		char *fullpath;
+		fullpath = build_full_cgpath_from_monitorpath(h, path, filename);
+
+		if (strcmp(filename, "io.weight") == 0 || strcmp(filename, "io.bfq.weight") == 0) {
+			if (!file_exists(fullpath)) {
+				free(path);
+				free(fullpath);
+				return 0;
+			}
+		}
+
+		ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
+		free(fullpath);
+	}
+	free(path);
+
+	return ret;
+}
+
+/* take devices cgroup line
+ *    /dev/foo rwx
+ * and convert it to a valid
+ *    type major:minor mode
+ * line. Return <0 on error. Dest is a preallocated buffer long enough to hold
+ * the output.
+ */
+static int device_cgroup_rule_parse_devpath(struct device_item *device,
+					    const char *devpath)
+{
+	__do_free char *path = NULL;
+	char *mode = NULL;
+	int n_parts, ret;
+	char *p;
+	struct stat sb;
+
+	path = must_copy_string(devpath);
+
+	/*
+	 * Read path followed by mode. Ignore any trailing text.
+	 * A '    # comment' would be legal. Technically other text is not
+	 * legal, we could check for that if we cared to.
+	 */
+	for (n_parts = 1, p = path; *p; p++) {
+		if (*p != ' ')
+			continue;
+		*p = '\0';
+
+		if (n_parts != 1)
+			break;
+		p++;
+		n_parts++;
+
+		while (*p == ' ')
+			p++;
+
+		mode = p;
+
+		if (*p == '\0')
+			return ret_set_errno(-1, EINVAL);
+	}
+
+	if (device_cgroup_parse_access(device, mode) < 0)
+		return -1;
+
+	if (n_parts == 1)
+		return ret_set_errno(-1, EINVAL);
+
+	ret = stat(path, &sb);
+	if (ret < 0)
+		return ret_set_errno(-1, errno);
+
+	mode_t m = sb.st_mode & S_IFMT;
+	switch (m) {
+	case S_IFBLK:
+		device->type = 'b';
+		break;
+	case S_IFCHR:
+		device->type = 'c';
+		break;
+	default:
+		return log_error_errno(-1, EINVAL, "Unsupported device type %i for \"%s\"", m, path);
+	}
+
+	device->major = MAJOR(sb.st_rdev);
+	device->minor = MINOR(sb.st_rdev);
+	device->allow = 1;
+	device->global_rule = LXC_BPF_DEVICE_CGROUP_LOCAL_RULE;
+
+	return 0;
+}
+
+static int convert_devpath(const char *invalue, char *dest)
+{
+	struct device_item device = {0};
+	int ret;
+
+	ret = device_cgroup_rule_parse_devpath(&device, invalue);
+	if (ret < 0)
+		return -1;
+
+	ret = snprintf(dest, 50, "%c %d:%d %s", device.type, device.major,
+		       device.minor, device.access);
+	if (ret < 0 || ret >= 50)
+		return log_error_errno(-1, ENAMETOOLONG, "Error on configuration value \"%c %d:%d %s\" (max 50 chars)",
+				       device.type, device.major, device.minor, device.access);
+
+	return 0;
+}
+
+/* Called from setup_limits - here we have the container's cgroup_data because
+ * we created the cgroups.
+ */
+static int isulad_cg_legacy_get_data(struct cgroup_ops *ops, const char *filename,
+                              char *value, size_t len)
+{
+	char *fullpath = NULL;
+	char *p = NULL;
+	struct hierarchy *h = NULL;
+	int ret = 0;
+	char *controller = NULL;
+
+	len = strlen(filename);
+	if (SIZE_MAX - 1 < len) {
+		errno = EINVAL;
+		return -1;
+	}
+	controller = calloc(1, len + 1);
+	if (controller == NULL) {
+		errno = ENOMEM;
+		return -1;
+	}
+	(void)strlcpy(controller, filename, len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+
+	h = get_hierarchy(ops, controller);
+	if (!h) {
+		ERROR("Failed to setup limits for the \"%s\" controller. "
+		      "The controller seems to be unused by \"cgfsng\" cgroup "
+		      "driver or not enabled on the cgroup hierarchy",
+		      controller);
+		errno = ENOENT;
+		free(controller);
+		return -ENOENT;
+	}
+
+	fullpath = must_make_path(h->container_full_path, filename, NULL);
+	ret = lxc_read_from_file(fullpath, value, len);
+	free(fullpath);
+	free(controller);
+	return ret;
+}
+
+static int isulad_cg_legacy_set_data(struct cgroup_ops *ops, const char *filename,
+                              const char *value)
+{
+	size_t len;
+	char *fullpath, *p;
+	/* "b|c <2^64-1>:<2^64-1> r|w|m" = 47 chars max */
+	char converted_value[50];
+	struct hierarchy *h;
+	int ret = 0;
+	char *controller = NULL;
+	int retry_count = 0;
+	int max_retry = 10;
+	char *container_cgroup = ops->container_cgroup;
+
+	len = strlen(filename);
+	controller = alloca(len + 1);
+	(void)strlcpy(controller, filename, len + 1);
+
+	p = strchr(controller, '.');
+	if (p)
+		*p = '\0';
+
+	if (strcmp("devices.allow", filename) == 0 && value[0] == '/') {
+		ret = convert_devpath(value, converted_value);
+		if (ret < 0)
+			return ret;
+		value = converted_value;
+	}
+
+	h = get_hierarchy(ops, controller);
+	if (!h) {
+		ERROR("Failed to setup limits for the \"%s\" controller. "
+		      "The controller seems to be unused by \"cgfsng\" cgroup "
+		      "driver or not enabled on the cgroup hierarchy",
+		      controller);
+		errno = ENOENT;
+		return -ENOENT;
+	}
+
+	fullpath = must_make_path(h->container_full_path, filename, NULL);
+
+retry:
+	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
+	if (ret != 0) {
+		if (retry_count < max_retry) {
+			SYSERROR("setting cgroup config for ready process caused \"failed to write %s to %s\".", value, fullpath);
+			(void)isulad_cg_legacy_handle_cpuset_hierarchy(h, container_cgroup);
+			(void)isulad_mkdir_eexist_on_last(h->container_full_path, 0755);
+			usleep(100 * 1000); /* 100 millisecond */
+			retry_count++;
+			goto retry;
+		}
+		lxc_write_error_message(ops->errfd,
+		                        "%s:%d: setting cgroup config for ready process caused failed to write %s to %s: %s",
+		                        __FILE__, __LINE__, value, fullpath, strerror(errno));
+	}
+	free(fullpath);
+	return ret;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_setup_limits_legacy(struct cgroup_ops *ops,
+						    struct lxc_conf *conf,
+						    bool do_devices)
+{
+	__do_free struct lxc_list *sorted_cgroup_settings = NULL;
+	struct lxc_list *cgroup_settings = &conf->cgroup;
+	struct lxc_list *iterator, *next;
+	struct lxc_cgroup *cg;
+	bool ret = false;
+	char value[21 + 1] = { 0 };
+	long long int readvalue, setvalue;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!conf)
+		return ret_set_errno(false, EINVAL);
+
+	cgroup_settings = &conf->cgroup;
+	if (lxc_list_empty(cgroup_settings))
+		return true;
+
+	if (!ops->hierarchies)
+		return ret_set_errno(false, EINVAL);
+
+	if (pure_unified_layout(ops))
+		return true;
+
+	sorted_cgroup_settings = sort_cgroup_settings(cgroup_settings);
+	if (!sorted_cgroup_settings)
+		return false;
+
+	lxc_list_for_each(iterator, sorted_cgroup_settings) {
+		cg = iterator->elem;
+
+		if (do_devices == !strncmp("devices", cg->subsystem, 7)) {
+			const char *cgvalue = cg->value;
+			if (strcmp(cg->subsystem, "files.limit") == 0) {
+				if (lxc_safe_long_long(cgvalue, &setvalue) != 0) {
+					SYSERROR("Invalid integer value %s", cgvalue);
+					goto out;
+				}
+				if (setvalue <= 0) {
+					cgvalue = "max";
+				}
+			}
+			if (isulad_cg_legacy_set_data(ops, cg->subsystem, cgvalue)) {
+				if (do_devices && (errno == EACCES || errno == EPERM)) {
+					SYSWARN("Failed to set \"%s\" to \"%s\"", cg->subsystem, cgvalue);
+					continue;
+				}
+				SYSERROR("Failed to set \"%s\" to \"%s\"", cg->subsystem, cgvalue);
+				goto out;
+			}
+			DEBUG("Set controller \"%s\" set to \"%s\"", cg->subsystem, cgvalue);
+		}
+
+		// isulad: check cpu shares
+		if (strcmp(cg->subsystem, "cpu.shares") == 0) {
+			if (isulad_cg_legacy_get_data(ops, cg->subsystem, value, sizeof(value) - 1) < 0) {
+				SYSERROR("Error get %s", cg->subsystem);
+				goto out;
+			}
+			trim(value);
+			if (lxc_safe_long_long(cg->value, &setvalue) != 0) {
+				SYSERROR("Invalid value %s", cg->value);
+				goto out;
+			}
+			if (lxc_safe_long_long(value, &readvalue) != 0) {
+				SYSERROR("Invalid value %s", value);
+				goto out;
+			}
+			if (setvalue > readvalue) {
+				ERROR("The maximum allowed cpu-shares is %s", value);
+				lxc_write_error_message(ops->errfd,
+				                        "%s:%d: setting cgroup config for ready process caused \"The maximum allowed cpu-shares is %s\".",
+				                        __FILE__, __LINE__, value);
+				goto out;
+			} else if (setvalue < readvalue) {
+				ERROR("The minimum allowed cpu-shares is %s", value);
+				lxc_write_error_message(ops->errfd,
+				                        "%s:%d: setting cgroup config for ready process caused \"The minimum allowed cpu-shares is %s\".",
+				                        __FILE__, __LINE__, value);
+				goto out;
+			}
+		}
+	}
+
+	ret = true;
+	INFO("Limits for the legacy cgroup hierarchies have been setup");
+out:
+	lxc_list_for_each_safe(iterator, sorted_cgroup_settings, next) {
+		lxc_list_del(iterator);
+		free(iterator);
+	}
+
+	return ret;
+}
+
+/*
+ * Some of the parsing logic comes from the original cgroup device v1
+ * implementation in the kernel.
+ */
+static int bpf_device_cgroup_prepare(struct cgroup_ops *ops,
+				     struct lxc_conf *conf, const char *key,
+				     const char *val)
+{
+#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
+	struct device_item device_item = {0};
+	int ret;
+
+	if (strcmp("devices.allow", key) == 0 && *val == '/')
+		ret = device_cgroup_rule_parse_devpath(&device_item, val);
+	else
+		ret = device_cgroup_rule_parse(&device_item, key, val);
+	if (ret < 0)
+		return log_error_errno(-1, EINVAL, "Failed to parse device string %s=%s", key, val);
+
+	ret = bpf_list_add_device(conf, &device_item);
+	if (ret < 0)
+		return -1;
+#endif
+	return 0;
+}
+
+__cgfsng_ops static bool isulad_cgfsng_setup_limits(struct cgroup_ops *ops,
+					     struct lxc_handler *handler)
+{
+	__do_free char *path = NULL;
+	struct lxc_list *cgroup_settings, *iterator;
+	struct hierarchy *h;
+	struct lxc_conf *conf;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, EINVAL);
+
+	if (!handler || !handler->conf)
+		return ret_set_errno(false, EINVAL);
+	conf = handler->conf;
+
+	if (lxc_list_empty(&conf->cgroup2))
+		return true;
+	cgroup_settings = &conf->cgroup2;
+
+	if (!pure_unified_layout(ops))
+		return true;
+
+	if (!ops->unified)
+		return false;
+	h = ops->unified;
+
+	lxc_list_for_each (iterator, cgroup_settings) {
+		struct lxc_cgroup *cg = iterator->elem;
+		int ret;
+
+		if (strncmp("devices", cg->subsystem, 7) == 0) {
+			ret = bpf_device_cgroup_prepare(ops, conf, cg->subsystem,
+							cg->value);
+		} else if (strcmp(cg->subsystem, "files.limit") == 0) {
+			long long int setvalue = 0;
+			const char *cgvalue = cg->value;
+
+			if (lxc_safe_long_long(cgvalue, &setvalue) != 0)
+				return log_error(false, "Invalid integer value %s", cgvalue);
+
+			if (setvalue <= 0)
+				cgvalue = "max";
+
+			ret = lxc_write_openat(h->container_full_path,
+					       cg->subsystem, cgvalue,
+					       strlen(cgvalue));
+			if (ret < 0)
+				return log_error_errno(false, errno, "Failed to set \"%s\" to \"%s\"",
+						       cg->subsystem, cgvalue);
+		} else {
+			if (strcmp(cg->subsystem, "io.weight") == 0 || strcmp(cg->subsystem, "io.bfq.weight") == 0) {
+				path = must_make_path(h->container_full_path, cg->subsystem, NULL);
+				if (!file_exists(path)) {
+					continue;
+				}
+			}
+			ret = lxc_write_openat(h->container_full_path,
+					       cg->subsystem, cg->value,
+					       strlen(cg->value));
+			if (ret < 0)
+				return log_error_errno(false, errno, "Failed to set \"%s\" to \"%s\"",
+						       cg->subsystem, cg->value);
+		}
+		TRACE("Set \"%s\" to \"%s\"", cg->subsystem, cg->value);
+	}
+
+	return log_info(true, "Limits for the unified cgroup hierarchy have been setup");
+}
+
+__cgfsng_ops bool isulad_cgfsng_devices_activate(struct cgroup_ops *ops,
+					  struct lxc_handler *handler)
+{
+#ifdef HAVE_STRUCT_BPF_CGROUP_DEV_CTX
+	__do_bpf_program_free struct bpf_program *devices = NULL;
+	int ret;
+	struct lxc_conf *conf;
+	struct hierarchy *unified;
+	struct lxc_list *it;
+	struct bpf_program *devices_old;
+
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+	if (!ops->hierarchies)
+		return true;
+
+	if (!ops->container_cgroup)
+		return ret_set_errno(false, EEXIST);
+
+	if (!handler || !handler->conf)
+		return ret_set_errno(false, EINVAL);
+	conf = handler->conf;
+
+	unified = ops->unified;
+	if (!unified || !unified->bpf_device_controller ||
+	    !unified->container_full_path || lxc_list_empty(&conf->devices))
+		return true;
+
+	devices = bpf_program_new(BPF_PROG_TYPE_CGROUP_DEVICE);
+	if (!devices)
+		return log_error_errno(false, ENOMEM, "Failed to create new bpf program");
+
+	ret = bpf_program_init(devices);
+	if (ret)
+		return log_error_errno(false, ENOMEM, "Failed to initialize bpf program");
+
+	lxc_list_for_each(it, &conf->devices) {
+		struct device_item *cur = it->elem;
+
+		ret = bpf_program_append_device(devices, cur);
+		if (ret)
+			return log_error_errno(false, ENOMEM, "Failed to add new rule to bpf device program: type %c, major %d, minor %d, access %s, allow %d, global_rule %d",
+					       cur->type,
+					       cur->major,
+					       cur->minor,
+					       cur->access,
+					       cur->allow,
+					       cur->global_rule);
+		TRACE("Added rule to bpf device program: type %c, major %d, minor %d, access %s, allow %d, global_rule %d",
+		      cur->type,
+		      cur->major,
+		      cur->minor,
+		      cur->access,
+		      cur->allow,
+		      cur->global_rule);
+	}
+
+	ret = bpf_program_finalize(devices);
+	if (ret)
+		return log_error_errno(false, ENOMEM, "Failed to finalize bpf program");
+
+	ret = bpf_program_cgroup_attach(devices, BPF_CGROUP_DEVICE,
+					unified->container_full_path,
+					BPF_F_ALLOW_MULTI);
+	if (ret)
+		return log_error_errno(false, ENOMEM, "Failed to attach bpf program");
+
+	/* Replace old bpf program. */
+	devices_old = move_ptr(conf->cgroup2_devices);
+	conf->cgroup2_devices = move_ptr(devices);
+	devices = move_ptr(devices_old);
+#endif
+	return true;
+}
+
+bool __cgfsng_delegate_controllers(struct cgroup_ops *ops, const char *cgroup)
+{
+	__do_free char *add_controllers = NULL, *base_path = NULL;
+	__do_free_string_list char **parts = NULL;
+	struct hierarchy *unified = ops->unified;
+	ssize_t parts_len;
+	char **it;
+	size_t full_len = 0;
+
+	if (!ops->hierarchies || !pure_unified_layout(ops) ||
+	    !unified->controllers[0])
+		return true;
+
+	/* For now we simply enable all controllers that we have detected by
+	 * creating a string like "+memory +pids +cpu +io".
+	 * TODO: In the near future we might want to support "-<controller>"
+	 * etc. but whether supporting semantics like this make sense will need
+	 * some thinking.
+	 */
+	for (it = unified->controllers; it && *it; it++) {
+		full_len += strlen(*it) + 2;
+		add_controllers = must_realloc(add_controllers, full_len + 1);
+
+		if (unified->controllers[0] == *it)
+			add_controllers[0] = '\0';
+
+		(void)strlcat(add_controllers, "+", full_len + 1);
+		(void)strlcat(add_controllers, *it, full_len + 1);
+
+		if ((it + 1) && *(it + 1))
+			(void)strlcat(add_controllers, " ", full_len + 1);
+	}
+
+	parts = lxc_string_split(cgroup, '/');
+	if (!parts)
+		return false;
+
+	parts_len = lxc_array_len((void **)parts);
+	if (parts_len > 0)
+		parts_len--;
+
+	base_path = must_make_path(unified->mountpoint, unified->container_base_path, NULL);
+	for (ssize_t i = -1; i < parts_len; i++) {
+		int ret;
+		__do_free char *target = NULL;
+
+		if (i >= 0)
+			base_path = must_append_path(base_path, parts[i], NULL);
+		target = must_make_path(base_path, "cgroup.subtree_control", NULL);
+		ret = lxc_writeat(-1, target, add_controllers, full_len);
+		if (ret < 0)
+			return log_error_errno(false, errno, "Could not enable \"%s\" controllers in the unified cgroup \"%s\"",
+					       add_controllers, target);
+		TRACE("Enable \"%s\" controllers in the unified cgroup \"%s\"", add_controllers, target);
+	}
+
+	return true;
+}
+
+__cgfsng_ops bool isulad_cgfsng_monitor_delegate_controllers(struct cgroup_ops *ops)
+{
+	return true;
+}
+
+__cgfsng_ops bool isulad_cgfsng_payload_delegate_controllers(struct cgroup_ops *ops)
+{
+	if (!ops)
+		return ret_set_errno(false, ENOENT);
+
+#ifdef HAVE_ISULAD
+	if (ops->no_controller) {
+		DEBUG("no controller found, isgnore isulad_cgfsng_payload_delegate_controllers");
+		return true;
+	}
+#endif
+
+	return __cgfsng_delegate_controllers(ops, ops->container_cgroup);
+}
+
+static bool cgroup_use_wants_controllers(const struct cgroup_ops *ops,
+				       char **controllers)
+{
+	if (!ops->cgroup_use)
+		return true;
+
+	for (char **cur_ctrl = controllers; cur_ctrl && *cur_ctrl; cur_ctrl++) {
+		bool found = false;
+
+		for (char **cur_use = ops->cgroup_use; cur_use && *cur_use; cur_use++) {
+			if (strcmp(*cur_use, *cur_ctrl) != 0)
+				continue;
+
+			found = true;
+			break;
+		}
+
+		if (found)
+			continue;
+
+		return false;
+	}
+
+	return true;
+}
+
+static void cg_unified_delegate(char ***delegate)
+{
+	__do_free char *buf = NULL;
+	char *standard[] = {"cgroup.subtree_control", "cgroup.threads", NULL};
+	char *token;
+	int idx;
+
+	buf = read_file("/sys/kernel/cgroup/delegate");
+	if (!buf) {
+		for (char **p = standard; p && *p; p++) {
+			idx = append_null_to_list((void ***)delegate);
+			(*delegate)[idx] = must_copy_string(*p);
+		}
+		SYSWARN("Failed to read /sys/kernel/cgroup/delegate");
+		return;
+	}
+
+	lxc_iterate_parts (token, buf, " \t\n") {
+		/*
+		 * We always need to chown this for both cgroup and
+		 * cgroup2.
+		 */
+		if (strcmp(token, "cgroup.procs") == 0)
+			continue;
+
+		idx = append_null_to_list((void ***)delegate);
+		(*delegate)[idx] = must_copy_string(token);
+	}
+}
+
+/* At startup, parse_hierarchies finds all the info we need about cgroup
+ * mountpoints and current cgroups, and stores it in @d.
+ */
+static int cg_hybrid_init(struct cgroup_ops *ops, bool relative, bool unprivileged)
+{
+	__do_free char *basecginfo = NULL, *line = NULL;
+	__do_free_string_list char **klist = NULL, **nlist = NULL;
+	__do_fclose FILE *f = NULL;
+	int ret;
+	size_t len = 0;
+
+	/* Root spawned containers escape the current cgroup, so use init's
+	 * cgroups as our base in that case.
+	 */
+	if (!relative && (geteuid() == 0))
+		basecginfo = read_file("/proc/1/cgroup");
+	else
+		basecginfo = read_file("/proc/self/cgroup");
+	if (!basecginfo)
+		return ret_set_errno(-1, ENOMEM);
+
+	ret = get_existing_subsystems(&klist, &nlist);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to retrieve available legacy cgroup controllers");
+
+	f = fopen("/proc/self/mountinfo", "re");
+	if (!f)
+		return log_error_errno(-1, errno, "Failed to open \"/proc/self/mountinfo\"");
+
+	lxc_cgfsng_print_basecg_debuginfo(basecginfo, klist, nlist);
+
+	while (getline(&line, &len, f) != -1) {
+		__do_free char *base_cgroup = NULL, *mountpoint = NULL;
+		__do_free_string_list char **controller_list = NULL;
+		int type;
+		struct hierarchy *new;
+
+		type = get_cgroup_version(line);
+		if (type == 0)
+			continue;
+
+		if (type == CGROUP2_SUPER_MAGIC && ops->unified)
+			continue;
+
+		if (ops->cgroup_layout == CGROUP_LAYOUT_UNKNOWN) {
+			if (type == CGROUP2_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_UNIFIED;
+			else if (type == CGROUP_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_LEGACY;
+		} else if (ops->cgroup_layout == CGROUP_LAYOUT_UNIFIED) {
+			if (type == CGROUP_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_HYBRID;
+		} else if (ops->cgroup_layout == CGROUP_LAYOUT_LEGACY) {
+			if (type == CGROUP2_SUPER_MAGIC)
+				ops->cgroup_layout = CGROUP_LAYOUT_HYBRID;
+		}
+
+		controller_list = cg_hybrid_get_controllers(klist, nlist, line, type);
+		if (!controller_list && type == CGROUP_SUPER_MAGIC)
+			continue;
+
+		if (type == CGROUP_SUPER_MAGIC)
+			if (controller_list_is_dup(ops->hierarchies, controller_list)) {
+				TRACE("Skipping duplicating controller");
+				continue;
+			}
+
+		mountpoint = cg_hybrid_get_mountpoint(line);
+		if (!mountpoint) {
+			WARN("Failed parsing mountpoint from \"%s\"", line);
+			continue;
+		}
+
+		if (type == CGROUP_SUPER_MAGIC)
+			base_cgroup = cg_hybrid_get_current_cgroup(basecginfo, controller_list[0], CGROUP_SUPER_MAGIC);
+		else
+			base_cgroup = cg_hybrid_get_current_cgroup(basecginfo, NULL, CGROUP2_SUPER_MAGIC);
+		if (!base_cgroup) {
+			WARN("Failed to find current cgroup");
+			continue;
+		}
+
+		trim(base_cgroup);
+		prune_init_scope(base_cgroup);
+
+		/* isulad: do not test writeable, if we run isulad in docker without cgroup namespace.
+		 * the base_cgroup will be docker/XXX.., mountpoint+base_cgroup may be not exist */
+
+		/*
+		 * reason:base cgroup may be started with /system.slice when cg_hybrid_init
+		 *	read /proc/1/cgroup on host, and cgroup init will set all containers
+		 *	cgroup path under /sys/fs/cgroup/<controller>/system.slice/xxx/lxc
+		 *	directory, this is not consistent with docker. The default cgroup path
+		 *	should be under /sys/fs/cgroup/<controller>/lxc directory.
+		 */
+
+		if (strlen(base_cgroup) > 1 && base_cgroup[0] == '/') {
+			base_cgroup[1] = '\0';
+		}
+
+		if (type == CGROUP2_SUPER_MAGIC) {
+			char *cgv2_ctrl_path;
+
+			cgv2_ctrl_path = must_make_path(mountpoint, base_cgroup,
+							"cgroup.controllers",
+							NULL);
+
+			controller_list = cg_unified_get_controllers(cgv2_ctrl_path);
+			free(cgv2_ctrl_path);
+			if (!controller_list) {
+				controller_list = cg_unified_make_empty_controller();
+				TRACE("No controllers are enabled for "
+				      "delegation in the unified hierarchy");
+			}
+		}
+
+		/* Exclude all controllers that cgroup use does not want. */
+		if (!cgroup_use_wants_controllers(ops, controller_list)) {
+			TRACE("Skipping controller");
+			continue;
+		}
+
+		new = add_hierarchy(&ops->hierarchies, move_ptr(controller_list), move_ptr(mountpoint), move_ptr(base_cgroup), type);
+		if (type == CGROUP2_SUPER_MAGIC && !ops->unified) {
+			if (unprivileged)
+				cg_unified_delegate(&new->cgroup2_chown);
+			ops->unified = new;
+		}
+	}
+
+	TRACE("Writable cgroup hierarchies:");
+	lxc_cgfsng_print_hierarchies(ops);
+
+	/* verify that all controllers in cgroup.use and all crucial
+	 * controllers are accounted for
+	 */
+	if (!all_controllers_found(ops))
+		return log_error_errno(-1, ENOENT, "Failed to find all required controllers");
+
+	return 0;
+}
+
+/* Get current cgroup from /proc/self/cgroup for the cgroupfs v2 hierarchy. */
+static char *cg_unified_get_current_cgroup(bool relative)
+{
+	__do_free char *basecginfo = NULL;
+	char *copy;
+	char *base_cgroup;
+
+	if (!relative && (geteuid() == 0))
+		basecginfo = read_file("/proc/1/cgroup");
+	else
+		basecginfo = read_file("/proc/self/cgroup");
+	if (!basecginfo)
+		return NULL;
+
+	base_cgroup = strstr(basecginfo, "0::/");
+	if (!base_cgroup)
+		return NULL;
+
+	base_cgroup = base_cgroup + 3;
+	copy = copy_to_eol(base_cgroup);
+	if (!copy)
+		return NULL;
+
+	return trim(copy);
+}
+
+static int cg_unified_init(struct cgroup_ops *ops, bool relative,
+			   bool unprivileged)
+{
+	__do_free char *subtree_path = NULL;
+	int ret;
+	char *mountpoint;
+	char **delegatable;
+	struct hierarchy *new;
+	char *base_cgroup = NULL;
+
+	ret = unified_cgroup_hierarchy();
+	if (ret == -ENOMEDIUM)
+		return ret_errno(ENOMEDIUM);
+
+	if (ret != CGROUP2_SUPER_MAGIC)
+		return 0;
+
+	base_cgroup = cg_unified_get_current_cgroup(relative);
+	if (!base_cgroup)
+		return ret_errno(EINVAL);
+	if (!relative)
+		prune_init_scope(base_cgroup);
+
+	/*
+	 * We assume that the cgroup we're currently in has been delegated to
+	 * us and we are free to further delege all of the controllers listed
+	 * in cgroup.controllers further down the hierarchy.
+	 */
+	mountpoint = must_copy_string(DEFAULT_CGROUP_MOUNTPOINT);
+	subtree_path = must_make_path(mountpoint, base_cgroup, "cgroup.controllers", NULL);
+	delegatable = cg_unified_get_controllers(subtree_path);
+	if (!delegatable)
+		delegatable = cg_unified_make_empty_controller();
+	if (!delegatable[0]) {
+		TRACE("No controllers are enabled for delegation");
+#ifdef HAVE_ISULAD
+		ops->no_controller = true;
+#endif
+	}
+
+	/* TODO: If the user requested specific controllers via lxc.cgroup.use
+	 * we should verify here. The reason I'm not doing it right is that I'm
+	 * not convinced that lxc.cgroup.use will be the future since it is a
+	 * global property. I much rather have an option that lets you request
+	 * controllers per container.
+	 */
+
+	new = add_hierarchy(&ops->hierarchies, delegatable, mountpoint, base_cgroup, CGROUP2_SUPER_MAGIC);
+	if (unprivileged)
+		cg_unified_delegate(&new->cgroup2_chown);
+
+	if (bpf_devices_cgroup_supported())
+		new->bpf_device_controller = 1;
+
+	ops->cgroup_layout = CGROUP_LAYOUT_UNIFIED;
+	ops->unified = new;
+
+	return CGROUP2_SUPER_MAGIC;
+}
+
+static int isulad_cg_init(struct cgroup_ops *ops, struct lxc_conf *conf)
+{
+	int ret;
+	const char *tmp;
+	bool relative = conf->cgroup_meta.relative;
+
+	tmp = lxc_global_config_value("lxc.cgroup.use");
+	if (tmp) {
+		__do_free char *pin = NULL;
+		char *chop, *cur;
+
+		pin = must_copy_string(tmp);
+		chop = pin;
+
+		lxc_iterate_parts(cur, chop, ",")
+			must_append_string(&ops->cgroup_use, cur);
+	}
+
+	ret = cg_unified_init(ops, relative, !lxc_list_empty(&conf->id_map));
+	if (ret < 0)
+		return -1;
+
+	if (ret == CGROUP2_SUPER_MAGIC)
+		return 0;
+
+	return cg_hybrid_init(ops, relative, !lxc_list_empty(&conf->id_map));
+}
+
+__cgfsng_ops static int isulad_cgfsng_data_init(struct cgroup_ops *ops, struct lxc_conf *conf)
+{
+	const char *cgroup_pattern;
+	const char *cgroup_tree;
+	__do_free char *container_cgroup = NULL, *__cgroup_tree = NULL;
+	size_t len;
+
+	if (!ops)
+		return ret_set_errno(-1, ENOENT);
+
+	/* copy system-wide cgroup information */
+	cgroup_pattern = lxc_global_config_value("lxc.cgroup.pattern");
+	if (cgroup_pattern && strcmp(cgroup_pattern, "") != 0)
+		ops->cgroup_pattern = must_copy_string(cgroup_pattern);
+
+	if (conf->cgroup_meta.dir) {
+		cgroup_tree = conf->cgroup_meta.dir;
+		container_cgroup = must_concat(&len, cgroup_tree, "/", conf->name, NULL);
+	} else if (ops->cgroup_pattern) {
+		__cgroup_tree = lxc_string_replace("%n", conf->name, ops->cgroup_pattern);
+		if (!__cgroup_tree)
+			return ret_set_errno(-1, ENOMEM);
+
+		cgroup_tree = __cgroup_tree;
+		container_cgroup = must_concat(&len, cgroup_tree, NULL);
+	} else {
+		cgroup_tree = NULL;
+		container_cgroup = must_concat(&len, conf->name, NULL);
+	}
+	if (!container_cgroup)
+		return ret_set_errno(-1, ENOMEM);
+
+	ops->container_cgroup = move_ptr(container_cgroup);
+
+	return 0;
+}
+
+struct cgroup_ops *cgfsng_ops_init(struct lxc_conf *conf)
+{
+	__do_free struct cgroup_ops *cgfsng_ops = NULL;
+
+	cgfsng_ops = malloc(sizeof(struct cgroup_ops));
+	if (!cgfsng_ops)
+		return ret_set_errno(NULL, ENOMEM);
+
+	memset(cgfsng_ops, 0, sizeof(struct cgroup_ops));
+	cgfsng_ops->cgroup_layout = CGROUP_LAYOUT_UNKNOWN;
+
+	if (isulad_cg_init(cgfsng_ops, conf))
+		return NULL;
+
+	cgfsng_ops->data_init = isulad_cgfsng_data_init;
+
+	cgfsng_ops->errfd = conf ? conf->errpipe[1] : -1;
+	cgfsng_ops->get_cgroup_full_path = isulad_cgfsng_get_cgroup_full_path;
+	cgfsng_ops->payload_destroy = isulad_cgfsng_payload_destroy;
+	cgfsng_ops->monitor_destroy = isulad_cgfsng_monitor_destroy;
+	cgfsng_ops->monitor_create = isulad_cgfsng_monitor_create;
+	cgfsng_ops->monitor_enter = isulad_cgfsng_monitor_enter;
+	cgfsng_ops->monitor_delegate_controllers = isulad_cgfsng_monitor_delegate_controllers;
+	cgfsng_ops->payload_delegate_controllers = isulad_cgfsng_payload_delegate_controllers;
+	cgfsng_ops->payload_create = isulad_cgfsng_payload_create;
+	cgfsng_ops->payload_enter = isulad_cgfsng_payload_enter;
+	cgfsng_ops->payload_finalize = isulad_cgfsng_payload_finalize;
+	cgfsng_ops->escape = isulad_cgfsng_escape;
+	cgfsng_ops->num_hierarchies = isulad_cgfsng_num_hierarchies;
+	cgfsng_ops->get_hierarchies = isulad_cgfsng_get_hierarchies;
+	cgfsng_ops->get_cgroup = isulad_cgfsng_get_cgroup;
+	cgfsng_ops->get = isulad_cgfsng_get;
+	cgfsng_ops->set = isulad_cgfsng_set;
+	cgfsng_ops->freeze = isulad_cgfsng_freeze;
+	cgfsng_ops->unfreeze = isulad_cgfsng_unfreeze;
+	cgfsng_ops->setup_limits_legacy = isulad_cgfsng_setup_limits_legacy;
+	cgfsng_ops->setup_limits = isulad_cgfsng_setup_limits;
+	cgfsng_ops->driver = "isulad_cgfsng";
+	cgfsng_ops->version = "1.0.0";
+	cgfsng_ops->attach = isulad_cgfsng_attach;
+	cgfsng_ops->chown = isulad_cgfsng_chown;
+	cgfsng_ops->mount = isulad_cgfsng_mount;
+	cgfsng_ops->devices_activate = isulad_cgfsng_devices_activate;
+
+	return move_ptr(cgfsng_ops);
+}
diff --git a/src/lxc/commands.c b/src/lxc/commands.c
index 07be1d5..2188b31 100644
--- a/src/lxc/commands.c
+++ b/src/lxc/commands.c
@@ -90,6 +90,10 @@ static const char *lxc_cmd_str(lxc_cmd_t cmd)
 		[LXC_CMD_GET_CGROUP_FD]			= "get_cgroup_fd",
 		[LXC_CMD_GET_LIMIT_CGROUP_FD]		= "get_limit_cgroup_fd",
 		[LXC_CMD_GET_SYSTEMD_SCOPE]		= "get_systemd_scope",
+#ifdef HAVE_ISULAD
+		[LXC_CMD_SET_TERMINAL_FIFOS]  		= "set_terminal_fifos",
+		[LXC_CMD_SET_TERMINAL_WINCH]  		= "set_terminal_winch",
+#endif
 	};
 
 	if (cmd >= LXC_CMD_MAX)
@@ -223,6 +227,12 @@ static ssize_t lxc_cmd_rsp_recv(int sock, struct lxc_cmd_rr *cmd)
 		break;
 	}
 
+#ifdef HAVE_ISULAD
+	/*isulad: add timeout 3s to avoid long block due to [lxc monitor] error*/
+	if (lxc_socket_set_timeout(sock, 3, 3) != 0) {
+		return syserror_ret(-1, "Failed to set timeout");
+	}
+#endif
 	/* Receive the first response including file descriptors if any. */
 	bytes_recv = lxc_cmd_rsp_recv_fds(sock, fds, rsp, cur_cmdstr);
 	if (bytes_recv < 0)
@@ -1576,7 +1586,12 @@ int lxc_cmd_serve_state_clients(const char *name, const char *lxcpath,
 
 	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
 	if (ret < 0)
+#ifdef HAVE_ISULAD
+		// for check connect failed, we just log warnning
+		return log_warn_errno(-1, errno, "Failed to serve state clients");
+#else
 		return log_error_errno(-1, errno, "Failed to serve state clients");
+#endif
 
 	return 0;
 }
@@ -1916,6 +1931,125 @@ static int lxc_cmd_rsp_send_enosys(int fd, int id)
 	return syserror_set(-ENOSYS, "Invalid command id %d", id);
 }
 
+#ifdef HAVE_ISULAD
+/*
+ * isulad: lxc_cmd_set_terminal_fifos: Set the fifos used for the container as terminal input/output
+ *
+ * @hashed_sock_name: hashed socket name
+ *
+ * Returns 0 when success, else when fail.
+ */
+int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath, const char *in_fifo,
+				const char *out_fifo, const char *err_fifo)
+{
+	int ret = 0;
+	bool stopped = false;
+	int len = 0;
+	char *tmp = NULL;
+	const char *split = "&&&&", *none_fifo_name = "none";
+	const char *cmd_in_fifo = in_fifo ? in_fifo : none_fifo_name;
+	const char *cmd_out_fifo = out_fifo ? out_fifo : none_fifo_name;
+	const char *cmd_err_fifo = err_fifo ? err_fifo : none_fifo_name;
+
+	if (len + strlen(cmd_in_fifo) + strlen(split) + strlen(cmd_out_fifo) +
+	     strlen(split) + strlen(cmd_err_fifo) == SIZE_MAX)
+		return -1;
+	len += strlen(cmd_in_fifo) + strlen(split) + strlen(cmd_out_fifo) + strlen(split) + strlen(cmd_err_fifo) + 1;
+	tmp = malloc(len);
+	if (tmp == NULL)
+		return -1;
+	ret = snprintf(tmp, len, "%s%s%s%s%s", cmd_in_fifo, split, cmd_out_fifo, split, cmd_err_fifo);
+	if (ret < 0 || ret >= len) {
+		ERROR("Failed to snprintf in fifo of command");
+		free(tmp);
+		return -1;
+	}
+
+	struct lxc_cmd_rr cmd = {
+		.req = {
+			.cmd = LXC_CMD_SET_TERMINAL_FIFOS,
+			.datalen = strlen(tmp)+1,
+			.data = tmp,
+		},
+	};
+
+	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
+	if (ret < 0) {
+		ERROR("Failed to send command to container");
+		free(tmp);
+		return -1;
+	}
+
+	if (cmd.rsp.ret != 0) {
+		ERROR("Command response error:%d", cmd.rsp.ret);
+		free(tmp);
+		return -1;
+	}
+
+	free(tmp);
+	return 0;
+}
+
+static int lxc_cmd_set_terminal_fifos_callback(int fd, struct lxc_cmd_req *req,
+					struct lxc_handler *handler, struct lxc_epoll_descr *descr)
+{
+	struct lxc_cmd_rsp rsp;
+	memset(&rsp, 0, sizeof(rsp));
+
+	rsp.ret = lxc_terminal_add_fifos(handler->conf, req->data);;
+
+	return lxc_cmd_rsp_send_reap(fd, &rsp);
+}
+
+struct lxc_cmd_set_terminal_winch_request {
+	unsigned int height;
+	unsigned int width;
+};
+
+int lxc_cmd_set_terminal_winch(const char *name, const char *lxcpath, unsigned int height, unsigned int width)
+{
+	int ret = 0;
+	bool stopped = false;
+	struct lxc_cmd_set_terminal_winch_request data = { 0 };
+
+	data.height = height;
+	data.width = width;
+
+	struct lxc_cmd_rr cmd = {
+		.req = {
+			.cmd = LXC_CMD_SET_TERMINAL_WINCH,
+			.datalen = sizeof(struct lxc_cmd_set_terminal_winch_request),
+			.data = &data,
+		},
+	};
+
+	ret = lxc_cmd(name, &cmd, &stopped, lxcpath, NULL);
+	if (ret < 0) {
+		ERROR("Failed to send command to container");
+		return -1;
+	}
+
+	if (cmd.rsp.ret != 0) {
+		ERROR("Command response error:%d", cmd.rsp.ret);
+		return -1;
+	}
+	return 0;
+}
+
+static int lxc_cmd_set_terminal_winch_callback(int fd, struct lxc_cmd_req *req,
+					struct lxc_handler *handler, struct lxc_epoll_descr *descr)
+{
+	struct lxc_cmd_rsp rsp;
+	struct lxc_cmd_set_terminal_winch_request *data = (struct lxc_cmd_set_terminal_winch_request *)(req->data);
+	memset(&rsp, 0, sizeof(rsp));
+
+	rsp.ret = lxc_set_terminal_winsz(&handler->conf->console, data->height, data->width);;
+
+	return lxc_cmd_rsp_send_reap(fd, &rsp);
+
+}
+#endif
+
 static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
 			   struct lxc_handler *handler,
 			   struct lxc_async_descr *descr)
@@ -1951,6 +2085,10 @@ static int lxc_cmd_process(int fd, struct lxc_cmd_req *req,
 		[LXC_CMD_GET_CGROUP_FD]			= lxc_cmd_get_cgroup_fd_callback,
 		[LXC_CMD_GET_LIMIT_CGROUP_FD]		= lxc_cmd_get_limit_cgroup_fd_callback,
 		[LXC_CMD_GET_SYSTEMD_SCOPE]		= lxc_cmd_get_systemd_scope_callback,
+#ifdef HAVE_ISULAD
+		[LXC_CMD_SET_TERMINAL_FIFOS]  			= lxc_cmd_set_terminal_fifos_callback,
+		[LXC_CMD_SET_TERMINAL_WINCH]  			= lxc_cmd_set_terminal_winch_callback,
+#endif
 	};
 
 	if (req->cmd >= LXC_CMD_MAX)
@@ -2104,6 +2242,46 @@ static int lxc_cmd_accept(int fd, uint32_t events, void *data,
 	return ret;
 }
 
+#ifdef HAVE_ISULAD
+// isulad required named unix socket to make safe connect,
+// so need to replace abstrace unix socket
+int lxc_server_init(const char *name, const char *lxcpath, const char *suffix)
+{
+	__do_close int fd = -EBADF;
+	int ret;
+	char path[LXC_AUDS_ADDR_LEN] = {0};
+	__do_free char *runtime_sock_dir = NULL;
+
+	runtime_sock_dir = generate_named_unix_sock_dir(name);
+	if (runtime_sock_dir == NULL)
+		return -1;
+
+	if (mkdir_p(runtime_sock_dir, 0700) < 0)
+		return log_error_errno(-1, errno, "Failed to create container runtime unix sock directory %s", path);
+
+	if (generate_named_unix_sock_path(name, suffix, path, sizeof(path)) != 0)
+		return -1;
+
+	fd = lxc_named_unix_open(path, SOCK_STREAM, 0);
+	if (fd < 0) {
+		if (errno == EADDRINUSE) {
+			WARN("Container \"%s\" appears to be already running", name);
+			(void)unlink(path);
+
+			fd = lxc_named_unix_open(path, SOCK_STREAM, 0);
+			if (fd < 0)
+				return log_error_errno(-1, errno, "Failed to create command socket %s", path);
+		} else
+			return log_error_errno(-1, errno, "Failed to create command socket %s", path);
+	}
+
+	ret = fcntl(fd, F_SETFD, FD_CLOEXEC);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to set FD_CLOEXEC on command socket file descriptor");
+
+	return log_trace(move_fd(fd), "Created unix socket \"%s\"", path);
+}
+#else
 int lxc_server_init(const char *name, const char *lxcpath, const char *suffix)
 {
 	__do_close int fd = -EBADF;
@@ -2128,6 +2306,7 @@ int lxc_server_init(const char *name, const char *lxcpath, const char *suffix)
 
 	return log_trace(move_fd(fd), "Created abstract unix socket \"%s\"", &path[1]);
 }
+#endif
 
 int lxc_cmd_mainloop_add(const char *name, struct lxc_async_descr *descr,
 			 struct lxc_handler *handler)
diff --git a/src/lxc/commands.h b/src/lxc/commands.h
index 2a39748..d5d1a0e 100644
--- a/src/lxc/commands.h
+++ b/src/lxc/commands.h
@@ -53,6 +53,10 @@ typedef enum {
 	LXC_CMD_GET_CGROUP_FD			= 24,
 	LXC_CMD_GET_LIMIT_CGROUP_FD		= 25,
 	LXC_CMD_GET_SYSTEMD_SCOPE		= 26,
+#ifdef HAVE_ISULAD
+	LXC_CMD_SET_TERMINAL_FIFOS = 27,
+	LXC_CMD_SET_TERMINAL_WINCH = 28,
+#endif
 	LXC_CMD_MAX,
 } lxc_cmd_t;
 
@@ -179,4 +183,10 @@ __hidden extern int lxc_cmd_get_limit_cgroup_fd(const char *name,
 						struct cgroup_fd *ret_fd);
 __hidden extern int lxc_cmd_get_devpts_fd(const char *name, const char *lxcpath);
 
+#ifdef HAVE_ISULAD
+__hidden extern int lxc_cmd_set_terminal_fifos(const char *name, const char *lxcpath,
+				const char *in_fifo, const char *out_fifo, const char *err_fifo);
+__hidden extern int lxc_cmd_set_terminal_winch(const char *name, const char *lxcpath, unsigned int height, unsigned int width);
+#endif
+
 #endif /* __commands_h */
diff --git a/src/lxc/commands_utils.c b/src/lxc/commands_utils.c
index 3558ff7..33780dd 100644
--- a/src/lxc/commands_utils.c
+++ b/src/lxc/commands_utils.c
@@ -143,12 +143,69 @@ int lxc_make_abstract_socket_name(char *path, size_t pathlen,
 	return 0;
 }
 
+#ifdef HAVE_ISULAD
+char *generate_named_unix_sock_dir(const char *name)
+{
+	__do_free char *exec_sock_dir = NULL;
+	__do_free char *rundir = NULL;
+
+	rundir = get_rundir();
+	if (!rundir)
+		rundir = strdup("/var/run");
+
+	if (asprintf(&exec_sock_dir, "%s/lxc/%s", rundir, name) < 0) {
+		return log_error_errno(NULL, errno, "Failed to allocate memory");
+	}
+
+	return move_ptr(exec_sock_dir);
+}
+
+int generate_named_unix_sock_path(const char *container_name, const char *sock_name,
+				char *out_path, size_t len)
+{
+#define MAX_SOCK_NAME_LENGTH 12
+	int ret;
+	__do_free char *sock_dir = NULL;
+	__do_free char *short_sock_name = NULL;
+
+	if (container_name == NULL || sock_name == NULL)
+		return -1;
+
+	sock_dir = generate_named_unix_sock_dir(container_name);
+	if (sock_dir == NULL)
+		return -1;
+
+	short_sock_name = strdup(sock_name);
+	if (strlen(short_sock_name) > MAX_SOCK_NAME_LENGTH)
+		short_sock_name[MAX_SOCK_NAME_LENGTH] = '\0';
+
+	ret = snprintf(out_path, len, "%s/%s.sock", sock_dir, short_sock_name);
+	if (ret < 0 || (size_t)ret >= len)
+		return log_error_errno(-1, errno, "Failed to allocate memory");
+
+	return 0;
+}
+#endif
+
 int lxc_cmd_connect(const char *name, const char *lxcpath,
 		    const char *hashed_sock_name, const char *suffix)
 {
 	int ret, client_fd;
 	char path[LXC_AUDS_ADDR_LEN] = {0};
 
+#ifdef HAVE_ISULAD
+	if (generate_named_unix_sock_path(name, suffix, path, sizeof(path)) != 0)
+		return -1;
+
+	if (file_exists(path)) {
+		client_fd = lxc_named_unix_connect(path);
+		if (client_fd < 0)
+			return -1;
+
+		return client_fd;
+	}
+#endif
+
 	ret = lxc_make_abstract_socket_name(path, sizeof(path), name, lxcpath,
 					    hashed_sock_name, suffix);
 	if (ret < 0)
diff --git a/src/lxc/commands_utils.h b/src/lxc/commands_utils.h
index 28ce490..2704ae2 100644
--- a/src/lxc/commands_utils.h
+++ b/src/lxc/commands_utils.h
@@ -67,4 +67,11 @@ __hidden extern int lxc_cmd_connect(const char *name, const char *lxcpath,
 __hidden extern void lxc_cmd_notify_state_listeners(const char *name,
                                                     const char *lxcpath,
                                                     lxc_state_t state);
+
+#ifdef HAVE_ISULAD
+__hidden extern char *generate_named_unix_sock_dir(const char *name);
+__hidden extern int generate_named_unix_sock_path(const char *container_name, const char *sock_name,
+												  char *out_path, size_t len);
+#endif /* HAVE ISULAD */
+
 #endif /* __LXC_COMMANDS_UTILS_H */
diff --git a/src/lxc/exec_commands.c b/src/lxc/exec_commands.c
new file mode 100644
index 0000000..bd81d66
--- /dev/null
+++ b/src/lxc/exec_commands.c
@@ -0,0 +1,477 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+ * Author: lifeng
+ * Create: 2019-12-08
+ * Description: provide container definition
+ * lxc: linux Container library
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+  ******************************************************************************/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+#include <caps.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <malloc.h>
+#include <poll.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/param.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+#include "af_unix.h"
+#include "cgroup.h"
+#include "exec_commands.h"
+#include "commands_utils.h"
+#include "conf.h"
+#include "config.h"
+#include "confile.h"
+#include "log.h"
+#include "lxc.h"
+#include "lxclock.h"
+#include "mainloop.h"
+#include "monitor.h"
+#include "string_utils.h"
+#include "terminal.h"
+#include "utils.h"
+
+lxc_log_define(commands_exec, lxc);
+
+static const char *lxc_exec_cmd_str(lxc_exec_cmd_t cmd)
+{
+	static const char *const cmdname[LXC_EXEC_CMD_MAX] = {
+		[LXC_EXEC_CMD_SET_TERMINAL_WINCH]  = "set_exec_terminal_winch",
+	};
+
+	if (cmd >= LXC_EXEC_CMD_MAX)
+		return "Invalid request";
+
+	return cmdname[cmd];
+}
+
+static int lxc_exec_cmd_rsp_recv(int sock, struct lxc_exec_cmd_rr *cmd)
+{
+	int ret, rspfd;
+	struct lxc_exec_cmd_rsp *rsp = &cmd->rsp;
+
+	/*isulad: add timeout 1s to avoid long block due to [lxc monitor] error*/
+	if (lxc_socket_set_timeout(sock, 1, 1) != 0) {
+		return syserror_ret(-1, "Failed to set timeout");
+	}
+
+	ret = lxc_cmd_rsp_recv_fds(sock, &rspfd, 1, rsp, sizeof(*rsp));
+	if (ret < 0) {
+		SYSERROR("Failed to receive response for command \"%s\"",
+		        lxc_exec_cmd_str(cmd->req.cmd));
+
+		if (errno == ECONNRESET || errno == EAGAIN || errno == EWOULDBLOCK) {
+			errno = ECONNRESET; /*isulad set errno ECONNRESET when timeout */
+			return -1;
+		}
+
+		return -1;
+	}
+	TRACE("Command \"%s\" received response", lxc_exec_cmd_str(cmd->req.cmd));
+
+	if (rsp->datalen == 0) {
+		DEBUG("Response data length for command \"%s\" is 0",
+		      lxc_exec_cmd_str(cmd->req.cmd));
+		return ret;
+	}
+
+	if (rsp->datalen > LXC_CMD_DATA_MAX) {
+		ERROR("Response data for command \"%s\" is too long: %d bytes > %d",
+		      lxc_exec_cmd_str(cmd->req.cmd), rsp->datalen, LXC_CMD_DATA_MAX);
+		return -1;
+	}
+
+	rsp->data = malloc(rsp->datalen);
+	if (!rsp->data) {
+		errno = ENOMEM;
+		ERROR("Failed to allocate response buffer for command \"%s\"",
+		      lxc_exec_cmd_str(cmd->req.cmd));
+		return -1;
+	}
+
+	ret = lxc_recv_nointr(sock, rsp->data, rsp->datalen, 0);
+	if (ret != rsp->datalen) {
+		SYSERROR("Failed to receive response data for command \"%s\"",
+		         lxc_exec_cmd_str(cmd->req.cmd));
+		return -1;
+	}
+
+	return ret;
+}
+
+static int lxc_exec_cmd_rsp_send(int fd, struct lxc_exec_cmd_rsp *rsp)
+{
+	ssize_t ret;
+
+	errno = EMSGSIZE;
+	ret = lxc_send_nointr(fd, rsp, sizeof(*rsp), MSG_NOSIGNAL);
+	if (ret < 0 || (size_t)ret != sizeof(*rsp)) {
+		SYSERROR("Failed to send command response %zd", ret);
+		return -1;
+	}
+
+	if (!rsp->data || rsp->datalen <= 0)
+		return 0;
+
+	errno = EMSGSIZE;
+	ret = lxc_send_nointr(fd, rsp->data, rsp->datalen, MSG_NOSIGNAL);
+	if (ret < 0 || ret != (ssize_t)rsp->datalen) {
+		SYSWARN("Failed to send command response data %zd", ret);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int lxc_exec_cmd_send(const char *name, struct lxc_exec_cmd_rr *cmd,
+			const char *lxcpath, const char *hashed_sock_name, const char *suffix)
+{
+	int client_fd, saved_errno;
+	ssize_t ret = -1;
+
+	client_fd = lxc_cmd_connect(name, lxcpath, hashed_sock_name, suffix);
+	if (client_fd < 0)
+		return -1;
+
+	ret = lxc_abstract_unix_send_credential(client_fd, &cmd->req,
+						sizeof(cmd->req));
+	if (ret < 0 || (size_t)ret != sizeof(cmd->req))
+		goto on_error;
+
+	if (cmd->req.datalen <= 0)
+		return client_fd;
+
+	errno = EMSGSIZE;
+	ret = lxc_send_nointr(client_fd, (void *)cmd->req.data,
+			      cmd->req.datalen, MSG_NOSIGNAL);
+	if (ret < 0 || ret != (ssize_t)cmd->req.datalen)
+		goto on_error;
+
+	return client_fd;
+
+on_error:
+	saved_errno = errno;
+	close(client_fd);
+	errno = saved_errno;
+
+	return -1;
+}
+
+static int lxc_exec_cmd(const char *name, struct lxc_exec_cmd_rr *cmd, const char *lxcpath, const char *hashed_sock_name, const char *suffix)
+{
+	int client_fd = -1;
+	int saved_errno;
+	int ret = -1;
+
+	client_fd = lxc_exec_cmd_send(name, cmd, lxcpath, hashed_sock_name, suffix);
+	if (client_fd < 0) {
+		SYSTRACE("Command \"%s\" failed to connect command socket",
+		         lxc_exec_cmd_str(cmd->req.cmd));
+		return -1;
+	}
+
+	ret = lxc_exec_cmd_rsp_recv(client_fd, cmd);
+
+	saved_errno = errno;
+	close(client_fd);
+	errno = saved_errno;
+	return ret;
+}
+
+int lxc_exec_cmd_set_terminal_winch(const char *name, const char *lxcpath, const char *suffix, unsigned int height, unsigned int width)
+{
+	int ret = 0;
+	struct lxc_exec_cmd_set_terminal_winch_request data = { 0 };
+
+	data.height = height;
+	data.width = width;
+
+	struct lxc_exec_cmd_rr cmd = {
+		.req = {
+			.cmd = LXC_EXEC_CMD_SET_TERMINAL_WINCH,
+			.datalen = sizeof(struct lxc_exec_cmd_set_terminal_winch_request),
+			.data = &data,
+		},
+	};
+
+	ret = lxc_exec_cmd(name, &cmd, lxcpath, NULL, suffix);
+	if (ret < 0) {
+		ERROR("Failed to send command to container");
+		return -1;
+	}
+
+	if (cmd.rsp.ret != 0) {
+		ERROR("Command response error:%d", cmd.rsp.ret);
+		return -1;
+	}
+	return 0;
+}
+
+static int lxc_exec_cmd_set_terminal_winch_callback(int fd, struct lxc_exec_cmd_req *req,
+					struct lxc_exec_command_handler *handler)
+{
+	struct lxc_exec_cmd_rsp rsp;
+	struct lxc_exec_cmd_set_terminal_winch_request *data = (struct lxc_exec_cmd_set_terminal_winch_request *)(req->data);
+	memset(&rsp, 0, sizeof(rsp));
+
+	rsp.ret = lxc_set_terminal_winsz(handler->terminal, data->height, data->width);;
+
+	return lxc_exec_cmd_rsp_send(fd, &rsp);
+
+}
+
+static int lxc_exec_cmd_process(int fd, struct lxc_exec_cmd_req *req,
+			   struct lxc_exec_command_handler *handler)
+{
+	typedef int (*callback)(int, struct lxc_exec_cmd_req *, struct lxc_exec_command_handler *);
+
+	callback cb[LXC_EXEC_CMD_MAX] = {
+		[LXC_EXEC_CMD_SET_TERMINAL_WINCH]  = lxc_exec_cmd_set_terminal_winch_callback,
+	};
+
+	if (req->cmd >= LXC_EXEC_CMD_MAX) {
+		ERROR("Undefined command id %d", req->cmd);
+		return -1;
+	}
+	return cb[req->cmd](fd, req, handler);
+}
+
+static void lxc_exec_cmd_fd_cleanup(int fd, struct lxc_epoll_descr *descr)
+{
+	lxc_mainloop_del_handler(descr, fd);
+	close(fd);
+	return;
+}
+
+static int lxc_exec_cmd_handler(int fd, uint32_t events, void *data,
+			   struct lxc_epoll_descr *descr)
+{
+	int ret;
+	struct lxc_exec_cmd_req req;
+	void *reqdata = NULL;
+	struct lxc_exec_command_handler *handler = data;
+
+	ret = lxc_abstract_unix_rcv_credential(fd, &req, sizeof(req));
+	if (ret < 0) {
+		SYSERROR("Failed to receive data on command socket for command "
+		         "\"%s\"", lxc_exec_cmd_str(req.cmd));
+
+		if (errno == EACCES) {
+			/* We don't care for the peer, just send and close. */
+			struct lxc_exec_cmd_rsp rsp = {.ret = ret};
+
+			lxc_exec_cmd_rsp_send(fd, &rsp);
+		}
+
+		goto out_close;
+	}
+
+	if (ret == 0)
+		goto out_close;
+
+	if (ret != sizeof(req)) {
+		WARN("Failed to receive full command request. Ignoring request "
+		     "for \"%s\"", lxc_exec_cmd_str(req.cmd));
+		ret = -1;
+		goto out_close;
+	}
+
+	if (req.datalen > LXC_CMD_DATA_MAX) {
+		ERROR("Received command data length %d is too large for "
+		      "command \"%s\"", req.datalen, lxc_exec_cmd_str(req.cmd));
+		errno = EFBIG;
+		ret = -EFBIG;
+		goto out_close;
+	}
+
+	if (req.datalen > 0) {
+		reqdata = alloca(req.datalen);
+		if (!reqdata) {
+			ERROR("Failed to allocate memory for \"%s\" command",
+			      lxc_exec_cmd_str(req.cmd));
+			errno = ENOMEM;
+			ret = -ENOMEM;
+			goto out_close;
+		}
+
+		ret = lxc_recv_nointr(fd, reqdata, req.datalen, 0);
+		if (ret != req.datalen) {
+			WARN("Failed to receive full command request. Ignoring "
+			     "request for \"%s\"", lxc_exec_cmd_str(req.cmd));
+			ret = LXC_MAINLOOP_ERROR;
+			goto out_close;
+		}
+
+		req.data = reqdata;
+	}
+
+	ret = lxc_exec_cmd_process(fd, &req, handler);
+	if (ret) {
+		/* This is not an error, but only a request to close fd. */
+		ret = LXC_MAINLOOP_CONTINUE;
+		goto out_close;
+	}
+
+out:
+	return ret;
+
+out_close:
+	lxc_exec_cmd_fd_cleanup(fd, descr);
+	goto out;
+}
+
+static int lxc_exec_cmd_accept(int fd, uint32_t events, void *data,
+			  struct lxc_epoll_descr *descr)
+{
+	int connection = -1;
+	int opt = 1, ret = -1;
+
+	connection = accept(fd, NULL, 0);
+	if (connection < 0) {
+		SYSERROR("Failed to accept connection to run command");
+		return LXC_MAINLOOP_ERROR;
+	}
+
+	ret = fcntl(connection, F_SETFD, FD_CLOEXEC);
+	if (ret < 0) {
+		SYSERROR("Failed to set close-on-exec on incoming command connection");
+		goto out_close;
+	}
+
+	ret = setsockopt(connection, SOL_SOCKET, SO_PASSCRED, &opt, sizeof(opt));
+	if (ret < 0) {
+		SYSERROR("Failed to enable necessary credentials on command socket");
+		goto out_close;
+	}
+
+	ret = lxc_mainloop_add_handler(descr, connection, lxc_exec_cmd_handler, data);
+	if (ret) {
+		ERROR("Failed to add command handler");
+		goto out_close;
+	}
+
+out:
+	return ret;
+
+out_close:
+	close(connection);
+	goto out;
+}
+#ifdef HAVE_ISULAD
+int lxc_exec_unix_sock_delete(const char *name, const char *suffix)
+{
+	char path[LXC_AUDS_ADDR_LEN] = {0};
+
+	if (name == NULL || suffix == NULL)
+		return -1;
+
+	if (generate_named_unix_sock_path(name, suffix, path, sizeof(path)) != 0)
+		return -1;
+
+	(void)unlink(path);
+
+	return 0;
+}
+
+int lxc_exec_cmd_init(const char *name, const char *lxcpath, const char *suffix)
+{
+	__do_close int fd = -EBADF;
+	int ret;
+	char path[LXC_AUDS_ADDR_LEN] = {0};
+	__do_free char *exec_sock_dir = NULL;
+
+	exec_sock_dir = generate_named_unix_sock_dir(name);
+	if (exec_sock_dir == NULL)
+		return -1;
+
+	if (mkdir_p(exec_sock_dir, 0600) < 0)
+		return log_error_errno(-1, errno, "Failed to create exec sock directory %s", path);
+
+	if (generate_named_unix_sock_path(name, suffix, path, sizeof(path)) != 0)
+		return -1;
+
+	TRACE("Creating unix socket \"%s\"", path);
+
+	fd = lxc_named_unix_open(path, SOCK_STREAM, 0);
+	if (fd < 0) {
+		if (errno == EADDRINUSE) {
+			WARN("Container \"%s\" exec unix sock is occupied", name);
+			(void)unlink(path);
+			fd = lxc_named_unix_open(path, SOCK_STREAM, 0);
+			if (fd < 0)
+				return log_error_errno(-1, errno, "Failed to create command socket %s", path);
+		} else {
+			return log_error_errno(-1, errno, "Failed to create command socket %s", path);
+		}
+	}
+
+	ret = fcntl(fd, F_SETFD, FD_CLOEXEC);
+	if (ret < 0)
+		return log_error_errno(-1, errno, "Failed to set FD_CLOEXEC on command socket file descriptor");
+
+	return log_trace(move_fd(fd), "Created unix socket \"%s\"", path);
+}
+#else
+int lxc_exec_cmd_init(const char *name, const char *lxcpath, const char *suffix)
+{
+	int fd, ret;
+	char path[LXC_AUDS_ADDR_LEN] = {0};
+
+	ret = lxc_make_abstract_socket_name(path, sizeof(path), name, lxcpath, NULL, suffix);
+	if (ret < 0)
+		return -1;
+	TRACE("Creating abstract unix socket \"%s\"", &path[1]);
+
+	fd = lxc_abstract_unix_open(path, SOCK_STREAM, 0);
+	if (fd < 0) {
+		SYSERROR("Failed to create command socket %s", &path[1]);
+		if (errno == EADDRINUSE)
+			ERROR("Container \"%s\" appears to be already running", name);
+
+		return -1;
+	}
+
+	ret = fcntl(fd, F_SETFD, FD_CLOEXEC);
+	if (ret < 0) {
+		SYSERROR("Failed to set FD_CLOEXEC on command socket file descriptor");
+		close(fd);
+		return -1;
+	}
+
+	return fd;
+}
+#endif
+
+int lxc_exec_cmd_mainloop_add(struct lxc_epoll_descr *descr, struct lxc_exec_command_handler *handler)
+{
+	int ret;
+	int fd = handler->maincmd_fd;
+
+	ret = lxc_mainloop_add_handler(descr, fd, lxc_exec_cmd_accept, handler);
+	if (ret < 0) {
+		ERROR("Failed to add handler for command socket");
+		close(fd);
+	}
+
+	return ret;
+}
diff --git a/src/lxc/exec_commands.h b/src/lxc/exec_commands.h
new file mode 100644
index 0000000..3ec2a22
--- /dev/null
+++ b/src/lxc/exec_commands.h
@@ -0,0 +1,77 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2019. All rights reserved.
+ * Author: lifeng
+ * Create: 2019-12-08
+ * Description: provide container definition
+ * lxc: linux Container library
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+  ******************************************************************************/
+
+#ifndef __LXC_EXEC_COMMANDS_H
+#define __LXC_EXEC_COMMANDS_H
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "lxccontainer.h"
+#include "macro.h"
+#include "state.h"
+#include "terminal.h"
+
+struct lxc_exec_command_handler {
+	int maincmd_fd;
+	struct lxc_terminal *terminal;
+};
+
+typedef enum {
+	LXC_EXEC_CMD_SET_TERMINAL_WINCH,
+	LXC_EXEC_CMD_MAX,
+} lxc_exec_cmd_t;
+
+struct lxc_exec_cmd_req {
+	lxc_exec_cmd_t cmd;
+	int datalen;
+	const void *data;
+};
+
+struct lxc_exec_cmd_rsp {
+	int ret; /* 0 on success, -errno on failure */
+	int datalen;
+	void *data;
+};
+
+struct lxc_exec_cmd_rr {
+	struct lxc_exec_cmd_req req;
+	struct lxc_exec_cmd_rsp rsp;
+};
+
+struct lxc_exec_cmd_set_terminal_winch_request {
+	unsigned int height;
+	unsigned int width;
+};
+
+struct lxc_epoll_descr;
+struct lxc_handler;
+
+extern int lxc_exec_cmd_init(const char *name, const char *lxcpath, const char *suffix);
+extern int lxc_exec_cmd_mainloop_add(struct lxc_epoll_descr *descr, struct lxc_exec_command_handler *handler);
+extern int lxc_exec_cmd_set_terminal_winch(const char *name, const char *lxcpath, const char *suffix, unsigned int height, unsigned int width);
+
+#ifdef HAVE_ISULAD
+extern int lxc_exec_unix_sock_delete(const char *name, const char *suffix);
+#endif
+
+#endif /* __exec_commands_h */
diff --git a/src/lxc/isulad_utils.c b/src/lxc/isulad_utils.c
new file mode 100644
index 0000000..ee39302
--- /dev/null
+++ b/src/lxc/isulad_utils.c
@@ -0,0 +1,535 @@
+/* SPDX-License-Identifier: LGPL-2.1+ */
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. Allrights reserved
+ * Description: isulad utils
+ * Author: lifeng
+ * Create: 2020-04-11
+******************************************************************************/
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <pwd.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <stdio_ext.h>
+
+#include "isulad_utils.h"
+#include "log.h"
+#include "path.h"
+#include "file_utils.h"
+
+lxc_log_define(isulad_utils, lxc);
+
+void *lxc_common_calloc_s(size_t size)
+{
+	if (size == 0 || size > SIZE_MAX) {
+		return NULL;
+	}
+
+	return calloc((size_t)1, size);
+}
+
+int lxc_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize)
+{
+	void *tmp = NULL;
+
+	if (newsize == 0) {
+		goto err_out;
+	}
+
+	tmp = lxc_common_calloc_s(newsize);
+	if (tmp == NULL) {
+		ERROR("Failed to malloc memory");
+		goto err_out;
+	}
+
+	if (oldptr != NULL) {
+		memcpy(tmp, oldptr, (newsize < oldsize) ? newsize : oldsize);
+
+		memset(oldptr, 0, oldsize);
+
+		free(oldptr);
+	}
+
+	*newptr = tmp;
+	return 0;
+
+err_out:
+	return -1;
+}
+
+char *safe_strdup(const char *src)
+{
+	char *dst = NULL;
+
+	if (src == NULL) {
+		return NULL;
+	}
+
+	dst = strdup(src);
+	if (dst == NULL) {
+		abort();
+	}
+
+	return dst;
+}
+
+int lxc_open(const char *filename, int flags, mode_t mode)
+{
+	char rpath[PATH_MAX] = {0x00};
+
+	if (cleanpath(filename, rpath, sizeof(rpath)) == NULL) {
+		return -1;
+	}
+	if (mode) {
+		return open(rpath, (int)((unsigned int)flags | O_CLOEXEC), mode);
+	} else {
+		return open(rpath, (int)((unsigned int)flags | O_CLOEXEC));
+	}
+}
+
+FILE *lxc_fopen(const char *filename, const char *mode)
+{
+	char rpath[PATH_MAX] = {0x00};
+
+	if (cleanpath(filename, rpath, sizeof(rpath)) == NULL) {
+		return NULL;
+	}
+
+	return fopen_cloexec(rpath, mode);
+}
+
+/* isulad: write error message */
+void lxc_write_error_message(int errfd, const char *format, ...)
+{
+	int ret;
+	char errbuf[BUFSIZ + 1] = {0};
+	ssize_t sret;
+	va_list argp;
+
+	if (errfd <= 0)
+		return;
+
+	va_start(argp, format);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
+	ret = vsnprintf(errbuf, BUFSIZ, format, argp);
+#pragma GCC diagnostic pop
+	va_end(argp);
+	if (ret < 0 || ret >= BUFSIZ)
+		SYSERROR("Failed to call vsnprintf");
+	sret = write(errfd, errbuf, strlen(errbuf));
+	if (sret < 0)
+		SYSERROR("Write errbuf failed");
+}
+
+/* isulad: read file to buffer */
+int lxc_file2str(const char *filename, char ret[], int cap)
+{
+    int fd, num_read;
+
+    if ((fd = lxc_open(filename, O_RDONLY | O_CLOEXEC, 0)) == -1)
+        return -1;
+    if ((num_read = read(fd, ret, cap - 1)) <= 0)
+        num_read = -1;
+    else
+        ret[num_read] = 0;
+    close(fd);
+
+    return num_read;
+}
+
+/* isuald: lxc_stat2proc() makes sure it can handle arbitrary executable file basenames
+ * for `cmd', i.e. those with embedded whitespace or embedded ')'s.
+ * Such names confuse %s (see scanf(3)), so the string is split and %39c
+ * is used instead. (except for embedded ')' "(%[^)]c)" would work.
+ */
+static proc_t *lxc_stat2proc(const char *S)
+{
+    int num;
+    proc_t *P = NULL;
+    char *tmp = NULL;
+
+    if (!S)
+        return NULL;
+
+    tmp = strrchr(S, ')');  /* split into "PID (cmd" and "<rest>" */
+    if (!tmp)
+        return NULL;
+    *tmp = '\0';            /* replace trailing ')' with NUL */
+
+    P = malloc(sizeof(proc_t));
+    if (P == NULL)
+        return NULL;
+    (void)memset(P, 0x00, sizeof(proc_t));
+
+    /* parse these two strings separately, skipping the leading "(". */
+    /* https://www.openwall.com/lists/musl/2013/11/15/5: musl's sscanf("%15c",cmd) 
+       requires exactly 15 characters; anything shorter is a matching failure. */
+#ifdef __MUSL__
+    num = sscanf(S, "%d (%15s", &P->pid, P->cmd);   /* comm[16] in kernel */
+#else
+    num = sscanf(S, "%d (%15c", &P->pid, P->cmd);   /* comm[16] in kernel */
+#endif
+    if (num != 2) {
+        ERROR("Call sscanf error: %s", errno ? strerror(errno) : "");
+        free(P);
+        return NULL;
+    }
+    num = sscanf(tmp + 2,           /* skip space after ')' too */
+                 "%c "
+                 "%d %d %d %d %d "
+                 "%lu %lu %lu %lu %lu "
+                 "%Lu %Lu %Lu %Lu "  /* utime stime cutime cstime */
+                 "%ld %ld %ld %ld "
+                 "%Lu "  /* start_time */
+                 "%lu "
+                 "%ld "
+                 "%lu %lu %lu %lu %lu %lu "
+                 "%*s %*s %*s %*s " /* discard, no RT signals & Linux 2.1 used hex */
+                 "%lu %lu %lu "
+                 "%d %d "
+                 "%lu %lu",
+                 &P->state,
+                 &P->ppid, &P->pgrp, &P->session, &P->tty, &P->tpgid,
+                 &P->flags, &P->min_flt, &P->cmin_flt, &P->maj_flt, &P->cmaj_flt,
+                 &P->utime, &P->stime, &P->cutime, &P->cstime,
+                 &P->priority, &P->nice, &P->timeout, &P->it_real_value,
+                 &P->start_time,
+                 &P->vsize,
+                 &P->rss,
+                 &P->rss_rlim, &P->start_code, &P->end_code, &P->start_stack, &P->kstk_esp,
+                 &P->kstk_eip,
+                 &P->wchan, &P->nswap, &P->cnswap,
+                 &P->exit_signal, &P->processor,  /* 2.2.1 ends with "exit_signal" */
+                 &P->rtprio, &P->sched  /* both added to 2.5.18 */
+                );
+    if (num != 35) {
+        ERROR("Call sscanf error: %s", errno ? strerror(errno) : "");
+        free(P);
+        return NULL;
+    }
+    if (P->tty == 0)
+        P->tty = -1;  /* the old notty val, update elsewhere bef. moving to 0 */
+    return P;
+}
+
+/* isulad: get starttime of process pid */
+unsigned long long lxc_get_process_startat(pid_t pid)
+{
+    int sret = 0;
+    unsigned long long startat = 0;
+    proc_t *pid_info = NULL;
+    char filename[PATH_MAX] = {0};
+    char sbuf[1024] = {0}; /* bufs for stat */
+
+    sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid);
+    if (sret < 0 || sret >= sizeof(filename)) {
+        ERROR("Failed to sprintf filename");
+        goto out;
+    }
+
+    if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) {
+        SYSERROR("Failed to read pidfile %s", filename);
+        goto out;
+    }
+
+    pid_info = lxc_stat2proc(sbuf);
+    if (!pid_info) {
+        ERROR("Failed to get proc stat info");
+        goto out;
+    }
+
+    startat = pid_info->start_time;
+out:
+    free(pid_info);
+    return startat;
+}
+
+// isulad: set env home in container
+int lxc_setup_env_home(uid_t uid)
+{
+#define __PASSWD_FILE__ "/etc/passwd"
+	char *homedir = "/"; // default home dir is /
+	FILE *stream = NULL;
+	struct passwd pw, *pwbufp = NULL;
+	char buf[BUFSIZ];
+	const char *curr_home = NULL;
+
+	curr_home = getenv("HOME");
+	// if user set or image set, just use it.
+	if (curr_home != NULL && strcmp(curr_home, "") != 0) {
+		return 0;
+	}
+
+	stream = fopen_cloexec(__PASSWD_FILE__, "r");
+	if (stream == NULL) {
+		SYSWARN("Failed to open %s", __PASSWD_FILE__);
+		goto set_env;
+	}
+
+#if IS_BIONIC
+	while (util_getpwent_r(stream, &pw, buf, sizeof(buf), &pwbufp) == 0 && pwbufp != NULL) {
+#else
+	while (fgetpwent_r(stream, &pw, buf, sizeof(buf), &pwbufp) == 0 && pwbufp != NULL) {
+#endif
+		if (pwbufp->pw_uid == uid) {
+			homedir = pwbufp->pw_dir;
+			goto set_env;
+		}
+	}
+	WARN("User invalid, can not find user '%u'", uid);
+
+set_env:
+	if (stream)
+		fclose(stream);
+
+	// if we didn't configure HOME, set it based on uid;
+	// override it if reach here.
+	if (setenv("HOME", homedir, 1) < 0) {
+		SYSERROR("Unable to set env 'HOME'");
+		return -1;
+	}
+
+	NOTICE("Setted env 'HOME' to %s", homedir);
+	return 0;
+}
+
+bool lxc_process_alive(pid_t pid, unsigned long long start_time)
+{
+	int sret = 0;
+	bool alive = true;
+	proc_t *pid_info = NULL;
+	char filename[PATH_MAX] = {0};
+	char sbuf[1024] = {0}; /* bufs for stat */
+
+	sret = kill(pid, 0);
+	if (sret < 0 && errno == ESRCH)
+		return false;
+
+	sret = snprintf(filename, sizeof(filename), "/proc/%d/stat", pid);
+	if (sret < 0 || sret >= sizeof(filename)) {
+		ERROR("Failed to sprintf filename");
+		goto out;
+	}
+
+	if ((lxc_file2str(filename, sbuf, sizeof(sbuf))) == -1) {
+		ERROR("Failed to read pidfile %s", filename);
+		alive = false;
+		goto out;
+	}
+
+	pid_info = lxc_stat2proc(sbuf);
+	if (!pid_info) {
+		ERROR("Failed to get proc stat info");
+		alive = false;
+		goto out;
+	}
+
+	if (start_time != pid_info->start_time)
+		alive = false;
+out:
+	free(pid_info);
+	return alive;
+}
+
+bool is_non_negative_num(const char *s)
+{
+	if (!s || !strcmp(s, ""))
+		return false;
+	while(*s  != '\0') {
+		if(!isdigit(*s))
+			return false;
+		++s;
+	}
+	return true;
+}
+
+static int hold_int(const char delim, bool required, char **src, unsigned int *dst)
+{
+    unsigned long long int res = 0;
+    char *err_str = NULL;
+
+    // ensure *src not a empty string
+    if (**src == '\0') {
+        ERROR("Empty subject on given entrie is not allowed.");
+        return -1;
+    }
+
+    errno = 0;
+    // covert string to long long
+    res = strtoull(*src, &err_str, 0);
+    if (errno != 0 && errno != ERANGE) {
+        ERROR("Parse int from string failed.");
+        return -1;
+    }
+
+    // **src is not a digit
+    if (err_str == *src) {
+        if (!required) {
+            ERROR("Integer part is missing.");
+            return -1;
+        }
+        // if required, just set 0
+        *dst = 0;
+    } else {
+        if (sizeof(void *) > 4 && res > UINT_MAX) { // make sure 64-bit platform behave same as 32-bit
+            res = UINT_MAX;
+        }
+        res = res & UINT_MAX;
+        *dst = (uint32_t)res;
+    }
+
+    // normal case
+    if (*err_str == delim) {
+        err_str++;
+    } else if (*err_str != '\0') {
+        ERROR("Invalid digit string.");
+        return -1;
+    }
+
+    *src = err_str; // update src to next valid context in line.
+    return 0;
+}
+
+static void hold_string(const char delim, char **src, char **dst)
+{
+    for (*dst = *src; **src != delim; ++(*src)) {
+        if (**src == '\0') {
+            break;
+        }
+    }
+
+    if (**src == delim) {
+        **src = '\0';
+        ++(*src);
+    }
+}
+
+static int parse_line_pw(const char delim, char *line, struct passwd *result)
+{
+    int ret = 0;
+    bool required = false;
+    char *walker = NULL;
+
+    walker = strpbrk(line, "\n");
+    if (walker != NULL) {
+        // clear newline char
+        *walker = '\0';
+    }
+
+    hold_string(delim, &line, &result->pw_name);
+
+    required = (result->pw_name[0] == '+' || result->pw_name[0] == '-') ? true : false;
+
+    hold_string(delim, &line, &result->pw_passwd);
+
+    ret = hold_int(delim, required, &line, &result->pw_uid);
+    if (ret != 0) {
+        // a legitimate line must have uid
+        ERROR("Parse uid error.");
+        return ret;
+    }
+
+    ret = hold_int(delim, required, &line, &result->pw_gid);
+    if (ret != 0) {
+        // it's ok to not provide gid
+        ERROR("Parse gid error.");
+        return ret;
+    }
+
+    hold_string(delim, &line, &result->pw_gecos);
+
+    hold_string(delim, &line, &result->pw_dir);
+
+    result->pw_shell = line;
+    return 0;
+}
+
+char *util_left_trim_space(char *str)
+{
+    char *begin = str;
+    char *tmp = str;
+    while (isspace(*begin)) {
+        begin++;
+    }
+    while ((*tmp++ = *begin++)) {
+    }
+    return str;
+}
+
+int util_getpwent_r(FILE *stream, struct passwd *resbuf, char *buffer, size_t buflen, struct passwd **result)
+{
+    const char delim = ':';
+    char *buff_end = NULL;
+    char *walker = NULL;
+    bool got = false;
+    int ret = 0;
+
+    if (stream == NULL || resbuf == NULL || buffer == NULL || result == NULL) {
+        ERROR("Password obj, params is NULL.");
+        return -1;
+    }
+
+    if (buflen <= 1) {
+        ERROR("Inadequate buffer length was given.");
+        return -1;
+    }
+
+    buff_end = buffer + buflen - 1;
+    flockfile(stream);
+
+    while (1) {
+        *buff_end = '\xff';
+        walker = fgets_unlocked(buffer, buflen, stream);
+        // if get NULL string
+        if (walker == NULL) {
+            *result = NULL;
+            // reach end of file, return error
+            if (feof(stream)) {
+                ret = ENOENT;
+                goto out;
+            }
+            // overflow buffer
+            ret = ERANGE;
+            goto out;
+        }
+        // just overflow last char in buffer
+        if (*buff_end != '\xff') {
+            *result = NULL;
+            ret = ERANGE;
+            goto out;
+        }
+
+        (void)util_left_trim_space(buffer);
+        // skip comment line and empty line
+        if (walker[0] == '#' || walker[0] == '\0') {
+            continue;
+        }
+
+        if (parse_line_pw(delim, walker, resbuf) == 0) {
+            got = true;
+            break;
+        }
+    }
+    if (!got) {
+        *result = NULL;
+        ret = ERANGE;
+        goto out;
+    }
+
+    *result = resbuf;
+    ret = 0;
+out:
+    funlockfile(stream);
+    return ret;
+}
diff --git a/src/lxc/isulad_utils.h b/src/lxc/isulad_utils.h
new file mode 100644
index 0000000..7a5eb89
--- /dev/null
+++ b/src/lxc/isulad_utils.h
@@ -0,0 +1,102 @@
+/* SPDX-License-Identifier: LGPL-2.1+ */
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. Allrights reserved
+ * Description: isulad utils
+ * Author: lifeng
+ * Create: 2020-04-11
+******************************************************************************/
+#ifndef __iSULAD_UTILS_H
+#define __iSULAD_UTILS_H
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <pwd.h>
+
+/* isulad: replace space with SPACE_MAGIC_STR */
+#define SPACE_MAGIC_STR "[#)"
+
+/* isulad:
+ ld cutime, cstime, priority, nice, timeout, it_real_value, rss,
+ c  state,
+ d  ppid, pgrp, session, tty, tpgid,
+ s  signal, blocked, sigignore, sigcatch,
+ lu flags, min_flt, cmin_flt, maj_flt, cmaj_flt, utime, stime,
+ lu rss_rlim, start_code, end_code, start_stack, kstk_esp, kstk_eip,
+ lu start_time, vsize, wchan, nswap, cnswap,
+*/
+
+/* Basic data structure which holds all information we can get about a process.
+ * (unless otherwise specified, fields are read from /proc/#/stat)
+ *
+ * Most of it comes from task_struct in linux/sched.h
+ */
+typedef struct proc_t {
+    // 1st 16 bytes
+    int pid;        /* process id */
+    int ppid;       /* pid of parent process */
+
+    char state;     /* single-char code for process state (S=sleeping) */
+
+    unsigned long long
+    utime,      /* user-mode CPU time accumulated by process */
+    stime,      /* kernel-mode CPU time accumulated by process */
+    // and so on...
+    cutime,     /* cumulative utime of process and reaped children */
+    cstime,     /* cumulative stime of process and reaped children */
+    start_time; /* start time of process -- seconds since 1-1-70 */
+
+    long
+    priority,   /* kernel scheduling priority */
+    timeout,    /* ? */
+    nice,       /* standard unix nice level of process */
+    rss,        /* resident set size from /proc/#/stat (pages) */
+    it_real_value;  /* ? */
+    unsigned long
+    rtprio,     /* real-time priority */
+    sched,      /* scheduling class */
+    vsize,      /* number of pages of virtual memory ... */
+    rss_rlim,   /* resident set size limit? */
+    flags,      /* kernel flags for the process */
+    min_flt,    /* number of minor page faults since process start */
+    maj_flt,    /* number of major page faults since process start */
+    cmin_flt,   /* cumulative min_flt of process and child processes */
+    cmaj_flt,   /* cumulative maj_flt of process and child processes */
+    nswap,      /* ? */
+    cnswap,     /* cumulative nswap ? */
+    start_code, /* address of beginning of code segment */
+    end_code,   /* address of end of code segment */
+    start_stack,    /* address of the bottom of stack for the process */
+    kstk_esp,   /* kernel stack pointer */
+    kstk_eip,   /* kernel instruction pointer */
+    wchan;      /* address of kernel wait channel proc is sleeping in */
+
+    char cmd[16];   /* basename of executable file in call to exec(2) */
+    int
+    pgrp,       /* process group id */
+    session,    /* session id */
+    tty,        /* full device number of controlling terminal */
+    tpgid,      /* terminal process group id */
+    exit_signal,    /* might not be SIGCHLD */
+    processor;      /* current (or most recent?) CPU */
+} proc_t;
+
+extern int lxc_mem_realloc(void **newptr, size_t newsize, void *oldptr, size_t oldsize);
+extern void *lxc_common_calloc_s(size_t size);
+extern char *safe_strdup(const char *src);
+
+extern int lxc_open(const char *filename, int flags, mode_t mode);
+extern FILE *lxc_fopen(const char *filename, const char *mode);
+
+extern void lxc_write_error_message(int errfd, const char *format, ...);
+extern int lxc_file2str(const char *filename, char ret[], int cap);
+extern int unsigned long long lxc_get_process_startat(pid_t pid);
+// set env home in container
+extern int lxc_setup_env_home(uid_t uid);
+
+extern bool lxc_process_alive(pid_t pid, unsigned long long start_time);
+
+extern bool is_non_negative_num(const char *s);
+
+int util_getpwent_r(FILE *stream, struct passwd *resbuf, char *buffer, size_t buflen, struct passwd **result);
+
+#endif
diff --git a/src/lxc/json/defs.c b/src/lxc/json/defs.c
new file mode 100644
index 0000000..4bf569a
--- /dev/null
+++ b/src/lxc/json/defs.c
@@ -0,0 +1,205 @@
+// Generated from defs.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "defs.h"
+
+defs_hook *make_defs_hook(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+    defs_hook *ret = NULL;
+    *err = 0;
+    if (tree == NULL)
+        return ret;
+    ret = safe_malloc(sizeof(*ret));
+    {
+        yajl_val val = get_val(tree, "path", yajl_t_string);
+        if (val != NULL) {
+            char *str = YAJL_GET_STRING(val);
+            ret->path = safe_strdup(str ? str : "");
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "args", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->args_len = YAJL_GET_ARRAY(tmp)->len;
+            if (YAJL_GET_ARRAY(tmp)->len > SIZE_MAX / sizeof(*ret->args) - 1) {
+                free_defs_hook(ret);
+                return NULL;
+            }
+            ret->args = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->args));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                if (val != NULL) {
+                    char *str = YAJL_GET_STRING(val);
+                    ret->args[i] = safe_strdup(str ? str : "");
+                }
+            }
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "env", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->env_len = YAJL_GET_ARRAY(tmp)->len;
+            if (YAJL_GET_ARRAY(tmp)->len > SIZE_MAX / sizeof(*ret->env) - 1) {
+                free_defs_hook(ret);
+                return NULL;
+            }
+            ret->env = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->env));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                if (val != NULL) {
+                    char *str = YAJL_GET_STRING(val);
+                    ret->env[i] = safe_strdup(str ? str : "");
+                }
+            }
+        }
+    }
+    {
+        yajl_val val = get_val(tree, "timeout", yajl_t_number);
+        if (val != NULL) {
+            int invalid = common_safe_int(YAJL_GET_NUMBER(val), (int *)&ret->timeout);
+            if (invalid) {
+                if (asprintf(err, "Invalid value '%s' with type 'integer' for key 'timeout': %s", YAJL_GET_NUMBER(val), strerror(-invalid)) < 0)
+                    *err = safe_strdup("error allocating memory");
+                free_defs_hook(ret);
+                return NULL;
+            }
+        }
+    }
+    if (ret->path == NULL) {
+        if (asprintf(err, "Required field '%s' not present", "path") < 0)
+            *err = safe_strdup("error allocating memory");
+        free_defs_hook(ret);
+        return NULL;
+    }
+
+    if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+        int i;
+        for (i = 0; i < tree->u.object.len; i++)
+            if (strcmp(tree->u.object.keys[i], "path") &&
+                strcmp(tree->u.object.keys[i], "args") &&
+                strcmp(tree->u.object.keys[i], "env") &&
+                strcmp(tree->u.object.keys[i], "timeout")) {
+                if (ctx->stderr > 0)
+                    fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+            }
+        }
+    return ret;
+}
+
+void free_defs_hook(defs_hook *ptr) {
+    if (ptr == NULL)
+        return;
+    free(ptr->path);
+    ptr->path = NULL;
+    if (ptr->args != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->args_len; i++) {
+            if (ptr->args[i] != NULL) {
+                free(ptr->args[i]);
+                ptr->args[i] = NULL;
+            }
+        }
+        free(ptr->args);
+        ptr->args = NULL;
+    }
+    if (ptr->env != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->env_len; i++) {
+            if (ptr->env[i] != NULL) {
+                free(ptr->env[i]);
+                ptr->env[i] = NULL;
+            }
+        }
+        free(ptr->env);
+        ptr->env = NULL;
+    }
+    free(ptr);
+}
+
+yajl_gen_status gen_defs_hook(yajl_gen g, defs_hook *ptr, struct parser_context *ctx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    *err = 0;
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->path != NULL)) {
+        char *str = "";
+        stat = reformat_map_key(g, "path", strlen("path"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->path != NULL) {
+            str = ptr->path;
+        }
+        stat = reformat_string(g, str, strlen(str));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->args != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "args", strlen("args"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->args != NULL) {
+            len = ptr->args_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = reformat_string(g, ptr->args[i], strlen(ptr->args[i]));
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->env != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "env", strlen("env"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->env != NULL) {
+            len = ptr->env_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = reformat_string(g, ptr->env[i], strlen(ptr->env[i]));
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->timeout)) {
+        long long int num = 0;
+        stat = reformat_map_key(g, "timeout", strlen("timeout"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->timeout) {
+            num = (long long int)ptr->timeout;
+        }
+        stat = reformat_int(g, num);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    return yajl_gen_status_ok;
+}
diff --git a/src/lxc/json/defs.h b/src/lxc/json/defs.h
new file mode 100644
index 0000000..0bbd8ac
--- /dev/null
+++ b/src/lxc/json/defs.h
@@ -0,0 +1,37 @@
+// Generated from defs.json. Do not edit!
+#ifndef DEFS_SCHEMA_H
+#define DEFS_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    char *path;
+
+    char **args;
+    size_t args_len;
+
+    char **env;
+    size_t env_len;
+
+    int timeout;
+
+}
+defs_hook;
+
+void free_defs_hook(defs_hook *ptr);
+
+defs_hook *make_defs_hook(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_defs_hook(yajl_gen g, defs_hook *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/json/json_common.c b/src/lxc/json/json_common.c
new file mode 100755
index 0000000..ec20c59
--- /dev/null
+++ b/src/lxc/json/json_common.c
@@ -0,0 +1,1153 @@
+// Auto generated file. Do not edit!
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <errno.h>
+#include <limits.h>
+#include "json_common.h"
+
+#define MAX_NUM_STR_LEN 21
+
+yajl_gen_status reformat_number(void *ctx, const char *str, size_t len) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_number(g, str, len);
+}
+
+yajl_gen_status reformat_uint(void *ctx, long long unsigned int num) {
+    char numstr[MAX_NUM_STR_LEN];
+    int ret;
+
+    ret = snprintf(numstr, MAX_NUM_STR_LEN, "%llu", num);
+    if (ret < 0 || ret >= MAX_NUM_STR_LEN) {
+        return yajl_gen_in_error_state;
+    }
+    return reformat_number(ctx, (const char *)numstr, strlen(numstr));
+}
+
+yajl_gen_status reformat_int(void *ctx, long long int num) {
+    char numstr[MAX_NUM_STR_LEN];
+    int ret;
+
+    ret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", num);
+    if (ret < 0 || ret >= MAX_NUM_STR_LEN) {
+        return yajl_gen_in_error_state;
+    }
+    return reformat_number(ctx, (const char *)numstr, strlen(numstr));
+}
+
+yajl_gen_status reformat_double(void *ctx, double num) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_double(g, num);
+}
+
+yajl_gen_status reformat_string(void *ctx, const char *str, size_t len) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_string(g, (const unsigned char *)str, len);
+}
+
+yajl_gen_status reformat_null(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_null(g);
+}
+
+yajl_gen_status reformat_bool(void *ctx, int boolean) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_bool(g, boolean);
+}
+
+yajl_gen_status reformat_map_key(void *ctx, const char *str, size_t len) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_string(g, (const unsigned char *)str, len);
+}
+
+yajl_gen_status reformat_start_map(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_map_open(g);
+}
+
+yajl_gen_status reformat_end_map(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_map_close(g);
+}
+
+yajl_gen_status reformat_start_array(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_array_open(g);
+}
+
+yajl_gen_status reformat_end_array(void *ctx) {
+    yajl_gen g = (yajl_gen) ctx;
+    return yajl_gen_array_close(g);
+}
+
+bool json_gen_init(yajl_gen *g, struct parser_context *ctx) {
+    *g = yajl_gen_alloc(NULL);
+    if (NULL == *g) {
+        return false;
+
+    }
+    yajl_gen_config(*g, yajl_gen_beautify, !(ctx->options & GEN_OPTIONS_SIMPLIFY));
+    yajl_gen_config(*g, yajl_gen_validate_utf8, !(ctx->options & GEN_OPTIONS_NOT_VALIDATE_UTF8));
+    return true;
+}
+
+yajl_val get_val(yajl_val tree, const char *name, yajl_type type) {
+    const char *path[] = { name, NULL };
+    return yajl_tree_get(tree, path, type);
+}
+
+void *safe_malloc(size_t size) {
+    void *ret = NULL;
+    if (size == 0) {
+        abort();
+    }
+    ret = calloc(1, size);
+    if (ret == NULL) {
+        abort();
+    }
+    return ret;
+}
+
+int common_safe_double(const char *numstr, double *converted) {
+    char *err_str = NULL;
+    double d;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    d = strtod(numstr, &err_str);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err_str == NULL || err_str == numstr || *err_str != '\0') {
+        return -EINVAL;
+    }
+
+    *converted = d;
+    return 0;
+}
+
+int common_safe_uint8(const char *numstr, uint8_t *converted) {
+    char *err = NULL;
+    unsigned long int uli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    uli = strtoul(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (uli > UINT8_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (uint8_t)uli;
+    return 0;
+}
+
+int common_safe_uint16(const char *numstr, uint16_t *converted) {
+    char *err = NULL;
+    unsigned long int uli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    uli = strtoul(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (uli > UINT16_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (uint16_t)uli;
+    return 0;
+}
+
+int common_safe_uint32(const char *numstr, uint32_t *converted) {
+    char *err = NULL;
+    unsigned long long int ull;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    ull = strtoull(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (ull > UINT32_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (uint32_t)ull;
+    return 0;
+}
+
+int common_safe_uint64(const char *numstr, uint64_t *converted) {
+    char *err = NULL;
+    unsigned long long int ull;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    ull = strtoull(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    *converted = (uint64_t)ull;
+    return 0;
+}
+
+int common_safe_uint(const char *numstr, unsigned int *converted) {
+    char *err = NULL;
+    unsigned long long int ull;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    ull = strtoull(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (ull > UINT_MAX) {
+        return -ERANGE;
+    }
+
+    *converted = (unsigned int)ull;
+    return 0;
+}
+
+int common_safe_int8(const char *numstr, int8_t *converted) {
+    char *err = NULL;
+    long int li;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    li = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (li > INT8_MAX || li < INT8_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int8_t)li;
+    return 0;
+}
+
+int common_safe_int16(const char *numstr, int16_t *converted) {
+    char *err = NULL;
+    long int li;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    li = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (li > INT16_MAX || li < INT16_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int16_t)li;
+    return 0;
+}
+
+int common_safe_int32(const char *numstr, int32_t *converted) {
+    char *err = NULL;
+    long long int lli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    lli = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (lli > INT32_MAX || lli < INT32_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int32_t)lli;
+    return 0;
+}
+
+int common_safe_int64(const char *numstr, int64_t *converted) {
+    char *err = NULL;
+    long long int lli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    lli = strtoll(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    *converted = (int64_t)lli;
+    return 0;
+}
+
+int common_safe_int(const char *numstr, int *converted) {
+    char *err = NULL;
+    long long int lli;
+
+    if (numstr == NULL) {
+        return -EINVAL;
+    }
+
+    errno = 0;
+    lli = strtol(numstr, &err, 0);
+    if (errno > 0) {
+        return -errno;
+    }
+
+    if (err == NULL || err == numstr || *err != '\0') {
+        return -EINVAL;
+    }
+
+    if (lli > INT_MAX || lli < INT_MIN) {
+        return -ERANGE;
+    }
+
+    *converted = (int)lli;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_int_int(void *ctx, json_map_int_int *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        char numstr[MAX_NUM_STR_LEN];
+        int nret;
+        nret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", (long long int)map->keys[i]);
+        if (nret < 0 || nret >= MAX_NUM_STR_LEN) {
+            if (!*err && asprintf(err, "Error to print string") < 0) {
+                *(err) = safe_strdup("error allocating memory");
+            }
+            return yajl_gen_in_error_state;
+        }
+        stat = reformat_string(g, numstr, strlen(numstr));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_int(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_int_int(json_map_int_int *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            // No need to free key for type int
+            // No need to free value for type int
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_int_int *make_json_map_int_int(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_int_int *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(int) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(int));
+        ret->values = safe_malloc((len + 1) * sizeof(int));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+
+            if (srckey != NULL) {
+                int invalid;
+                invalid = common_safe_int(srckey, &(ret->keys[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid key '%s' with type 'int': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_int(ret);
+                    return NULL;
+                }
+            }
+
+            if (srcval != NULL) {
+                int invalid;
+                if (!YAJL_IS_NUMBER(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_int(ret);
+                    return NULL;
+                }
+                invalid = common_safe_int(YAJL_GET_NUMBER(srcval), &(ret->values[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_int(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_int_int(json_map_int_int *map, int key, int val) {
+    size_t len;
+    int *keys = NULL;
+    int *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(int) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(int));
+    vals = safe_malloc(len * sizeof(int));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(int));
+        (void)memcpy(vals, map->values, map->len * sizeof(int));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = key;
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_int_bool(void *ctx, json_map_int_bool *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        char numstr[MAX_NUM_STR_LEN];
+        int nret;
+        nret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", (long long int)map->keys[i]);
+        if (nret < 0 || nret >= MAX_NUM_STR_LEN) {
+            if (!*err && asprintf(err, "Error to print string") < 0) {
+                *(err) = safe_strdup("error allocating memory");
+            }
+            return yajl_gen_in_error_state;
+        }
+        stat = reformat_string(g, numstr, strlen(numstr));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_bool(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_int_bool(json_map_int_bool *map) {
+    if (map != NULL) {
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_int_bool *make_json_map_int_bool(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_int_bool *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(int) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(int));
+        ret->values = safe_malloc((len + 1) * sizeof(bool));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+
+            if (srckey != NULL) {
+                int invalid;
+                invalid = common_safe_int(srckey, &(ret->keys[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid key '%s' with type 'int': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_bool(ret);
+                    return NULL;
+                }
+            }
+
+            if (srcval != NULL) {
+                if (YAJL_IS_TRUE(srcval)) {
+                    ret->values[i] = true;
+                } else if (YAJL_IS_FALSE(srcval)) {
+                    ret->values[i] = false;
+                } else {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'bool' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_bool(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_int_bool(json_map_int_bool *map, int key, bool val) {
+    size_t len;
+    int *keys = NULL;
+    bool *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(int) - 1) < map->len || (SIZE_MAX / sizeof(bool) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(int));
+    vals = safe_malloc(len * sizeof(bool));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(int));
+        (void)memcpy(vals, map->values, map->len * sizeof(bool));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = key;
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_int_string(void *ctx, json_map_int_string *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        char numstr[MAX_NUM_STR_LEN];
+        int nret;
+        nret = snprintf(numstr, MAX_NUM_STR_LEN, "%lld", (long long int)map->keys[i]);
+        if (nret < 0 || nret >= MAX_NUM_STR_LEN) {
+            if (!*err && asprintf(err, "Error to print string") < 0) {
+                *(err) = safe_strdup("error allocating memory");
+            }
+            return yajl_gen_in_error_state;
+        }
+        stat = reformat_string(g, numstr, strlen(numstr));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_string(g, map->values[i], strlen(map->values[i]));;
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_int_string(json_map_int_string *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            // No need to free key for type int
+            free(map->values[i]);
+            map->values[i] = NULL;
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_int_string *make_json_map_int_string(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_int_string *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(int));
+        ret->values = safe_malloc((len + 1) * sizeof(char *));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+
+            if (srckey != NULL) {
+                int invalid;
+                invalid = common_safe_int(srckey, &(ret->keys[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid key '%s' with type 'int': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_string(ret);
+                    return NULL;
+                }
+            }
+
+            if (srcval != NULL) {
+                if (!YAJL_IS_STRING(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'string' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_int_string(ret);
+                    return NULL;
+                }
+                char *str = YAJL_GET_STRING(srcval);
+                ret->values[i] = safe_strdup(str ? str : "");
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_int_string(json_map_int_string *map, int key, const char *val) {
+    size_t len;
+    int *keys = NULL;
+    char **vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(int) - 1) < map->len || (SIZE_MAX / sizeof(char *) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(int));
+    vals = safe_malloc(len * sizeof(char *));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(int));
+        (void)memcpy(vals, map->values, map->len * sizeof(char *));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = key;
+    map->values[map->len] = safe_strdup(val ? val : "");
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_string_int(void *ctx, json_map_string_int *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        stat = reformat_string(g, map->keys[i], strlen(map->keys[i]));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_int(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_string_int(json_map_string_int *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            free(map->keys[i]);
+            map->keys[i] = NULL;
+            // No need to free value for type int
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_string_int *make_json_map_string_int(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_string_int *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(char *));
+        ret->values = safe_malloc((len + 1) * sizeof(int));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+            ret->keys[i] = safe_strdup(srckey ? srckey : "");
+
+            if (srcval != NULL) {
+                int invalid;
+                if (!YAJL_IS_NUMBER(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_int(ret);
+                    return NULL;
+                }
+                invalid = common_safe_int(YAJL_GET_NUMBER(srcval), &(ret->values[i]));
+                if (invalid) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'int' for key '%s': %s", srckey, strerror(-invalid)) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_int(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_string_int(json_map_string_int *map, const char *key, int val) {
+    size_t len;
+    char **keys = NULL;
+    int *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(char *) - 1) < map->len || (SIZE_MAX / sizeof(int) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(char *));
+    vals = safe_malloc(len * sizeof(int));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(char *));
+        (void)memcpy(vals, map->values, map->len * sizeof(int));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = safe_strdup(key ? key : "");
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_string_bool(void *ctx, json_map_string_bool *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        stat = reformat_string(g, map->keys[i], strlen(map->keys[i]));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_bool(g, map->values[i]);
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_string_bool(json_map_string_bool *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            free(map->keys[i]);
+            map->keys[i] = NULL;
+            // No need to free value for type bool
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_string_bool *make_json_map_string_bool(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_string_bool *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(char *));
+        ret->values = safe_malloc((len + 1) * sizeof(bool));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+            ret->keys[i] = safe_strdup(srckey ? srckey : "");
+
+            if (srcval != NULL) {
+                if (YAJL_IS_TRUE(srcval)) {
+                    ret->values[i] = true;
+                } else if (YAJL_IS_FALSE(srcval)) {
+                    ret->values[i] = false;
+                } else {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'bool' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_bool(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    return ret;
+}
+
+int append_json_map_string_bool(json_map_string_bool *map, const char *key, bool val) {
+    size_t len;
+    char **keys = NULL;
+    bool *vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    if ((SIZE_MAX / sizeof(char *) - 1) < map->len || (SIZE_MAX / sizeof(bool) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(char *));
+    vals = safe_malloc(len * sizeof(bool));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(char *));
+        (void)memcpy(vals, map->values, map->len * sizeof(bool));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = safe_strdup(key ? key : "");
+    map->values[map->len] = val;
+
+    map->len++;
+    return 0;
+}
+
+yajl_gen_status gen_json_map_string_string(void *ctx, json_map_string_string *map, struct parser_context *ptx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    yajl_gen g = (yajl_gen) ctx;
+    size_t len = 0, i = 0;
+    if (map != NULL) {
+        len = map->len;
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 0);
+    }
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+
+    }
+    for (i = 0; i < len; i++) {
+        stat = reformat_string(g, map->keys[i], strlen(map->keys[i]));
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_string(g, map->values[i], strlen(map->values[i]));;
+        if (yajl_gen_status_ok != stat) {
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+    }
+
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat) {
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if (!len && !(ptx->options & GEN_OPTIONS_SIMPLIFY)) {
+        yajl_gen_config(g, yajl_gen_beautify, 1);
+    }
+    return yajl_gen_status_ok;
+}
+
+void free_json_map_string_string(json_map_string_string *map) {
+    if (map != NULL) {
+        size_t i;
+        for (i = 0; i < map->len; i++) {
+            free(map->keys[i]);
+            map->keys[i] = NULL;
+            free(map->values[i]);
+            map->values[i] = NULL;
+        }
+        free(map->keys);
+        map->keys = NULL;
+        free(map->values);
+        map->values = NULL;
+        free(map);
+    }
+}
+json_map_string_string *make_json_map_string_string(yajl_val src, struct parser_context *ctx, parser_error *err) {
+    json_map_string_string *ret = NULL;
+    if (src != NULL && YAJL_GET_OBJECT(src) != NULL) {
+        size_t i;
+        size_t len = YAJL_GET_OBJECT(src)->len;
+        if (len > SIZE_MAX / sizeof(char *) - 1) {
+            return NULL;
+        }
+        ret = safe_malloc(sizeof(*ret));
+        ret->len = len;
+        ret->keys = safe_malloc((len + 1) * sizeof(char *));
+        ret->values = safe_malloc((len + 1) * sizeof(char *));
+        for (i = 0; i < len; i++) {
+            const char *srckey = YAJL_GET_OBJECT(src)->keys[i];
+            yajl_val srcval = YAJL_GET_OBJECT(src)->values[i];
+            ret->keys[i] = safe_strdup(srckey ? srckey : "");
+
+            if (srcval != NULL) {
+                if (!YAJL_IS_STRING(srcval)) {
+                    if (*err == NULL && asprintf(err, "Invalid value with type 'string' for key '%s'", srckey) < 0) {
+                        *(err) = safe_strdup("error allocating memory");
+                    }
+                    free_json_map_string_string(ret);
+                    return NULL;
+                }
+                char *str = YAJL_GET_STRING(srcval);
+                ret->values[i] = safe_strdup(str ? str : "");
+            }
+        }
+    }
+    return ret;
+}
+int append_json_map_string_string(json_map_string_string *map, const char *key, const char *val) {
+    size_t len, i;
+    char **keys = NULL;
+    char **vals = NULL;
+
+    if (map == NULL) {
+        return -1;
+    }
+
+    for (i = 0; i < map->len; i++) {
+        if (strcmp(map->keys[i], key) == 0) {
+            free(map->values[i]);
+            map->values[i] = safe_strdup(val ? val : "");
+            return 0;
+        }
+    }
+
+    if ((SIZE_MAX / sizeof(char *) - 1) < map->len) {
+        return -1;
+    }
+
+    len = map->len + 1;
+    keys = safe_malloc(len * sizeof(char *));
+    vals = safe_malloc(len * sizeof(char *));
+
+    if (map->len) {
+        (void)memcpy(keys, map->keys, map->len * sizeof(char *));
+        (void)memcpy(vals, map->values, map->len * sizeof(char *));
+    }
+    free(map->keys);
+    map->keys = keys;
+    free(map->values);
+    map->values = vals;
+    map->keys[map->len] = safe_strdup(key ? key : "");
+    map->values[map->len] = safe_strdup(val ? val : "");
+
+    map->len++;
+    return 0;
+}
diff --git a/src/lxc/json/json_common.h b/src/lxc/json/json_common.h
new file mode 100755
index 0000000..bcac13e
--- /dev/null
+++ b/src/lxc/json/json_common.h
@@ -0,0 +1,194 @@
+// Auto generated file. Do not edit!
+#ifndef _JSON_COMMON_H
+#define _JSON_COMMON_H
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <yajl/yajl_tree.h>
+#include <yajl/yajl_gen.h>
+#include "isulad_utils.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+# undef linux
+
+#ifdef __MUSL__
+#undef stdin
+#undef stdout
+#undef stderr
+#define stdin stdin
+#define stdout stdout
+#define stderr stderr
+#endif
+
+//options to report error if there is unknown key found in json
+# define PARSE_OPTIONS_STRICT 0x01
+//options to generate all key and value
+# define GEN_OPTIONS_ALLKEYVALUE 0x02
+//options to generate simplify(no indent) json string
+# define GEN_OPTIONS_SIMPLIFY 0x04
+//options not to validate utf8 data
+# define GEN_OPTIONS_NOT_VALIDATE_UTF8 0x08
+
+#define GEN_SET_ERROR_AND_RETURN(stat, err) { \
+        if (*(err) == NULL) {\
+            if (asprintf(err, "%s: %s: %d: error generating json, errcode: %d", __FILE__, __func__, __LINE__, stat) < 0) { \
+                *(err) = safe_strdup("error allocating memory"); \
+            } \
+        }\
+        return stat; \
+    }
+
+typedef char *parser_error;
+
+struct parser_context {
+    unsigned int options;
+    FILE *stderr;
+};
+
+yajl_gen_status reformat_number(void *ctx, const char *str, size_t len);
+
+yajl_gen_status reformat_uint(void *ctx, long long unsigned int num);
+
+yajl_gen_status reformat_int(void *ctx, long long int num);
+
+yajl_gen_status reformat_double(void *ctx, double num);
+
+yajl_gen_status reformat_string(void *ctx, const char *str, size_t len);
+
+yajl_gen_status reformat_null(void *ctx);
+
+yajl_gen_status reformat_bool(void *ctx, int boolean);
+
+yajl_gen_status reformat_map_key(void *ctx, const char *str, size_t len);
+
+yajl_gen_status reformat_start_map(void *ctx);
+
+yajl_gen_status reformat_end_map(void *ctx);
+
+yajl_gen_status reformat_start_array(void *ctx);
+
+yajl_gen_status reformat_end_array(void *ctx);
+
+bool json_gen_init(yajl_gen *g, struct parser_context *ctx);
+
+yajl_val get_val(yajl_val tree, const char *name, yajl_type type);
+
+void *safe_malloc(size_t size);
+
+int common_safe_double(const char *numstr, double *converted);
+
+int common_safe_uint8(const char *numstr, uint8_t *converted);
+
+int common_safe_uint16(const char *numstr, uint16_t *converted);
+
+int common_safe_uint32(const char *numstr, uint32_t *converted);
+
+int common_safe_uint64(const char *numstr, uint64_t *converted);
+
+int common_safe_uint(const char *numstr, unsigned int *converted);
+
+int common_safe_int8(const char *numstr, int8_t *converted);
+
+int common_safe_int16(const char *numstr, int16_t *converted);
+
+int common_safe_int32(const char *numstr, int32_t *converted);
+
+int common_safe_int64(const char *numstr, int64_t *converted);
+
+int common_safe_int(const char *numstr, int *converted);
+
+typedef struct {
+    int *keys;
+    int *values;
+    size_t len;
+} json_map_int_int;
+
+void free_json_map_int_int(json_map_int_int *map);
+
+json_map_int_int *make_json_map_int_int(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_int_int(void *ctx, json_map_int_int *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_int_int(json_map_int_int *map, int key, int val);
+
+typedef struct {
+    int *keys;
+    bool *values;
+    size_t len;
+} json_map_int_bool;
+
+void free_json_map_int_bool(json_map_int_bool *map);
+
+json_map_int_bool *make_json_map_int_bool(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_int_bool(void *ctx, json_map_int_bool *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_int_bool(json_map_int_bool *map, int key, bool val);
+
+typedef struct {
+    int *keys;
+    char **values;
+    size_t len;
+} json_map_int_string;
+
+void free_json_map_int_string(json_map_int_string *map);
+
+json_map_int_string *make_json_map_int_string(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_int_string(void *ctx, json_map_int_string *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_int_string(json_map_int_string *map, int key, const char *val);
+
+typedef struct {
+    char **keys;
+    int *values;
+    size_t len;
+} json_map_string_int;
+
+void free_json_map_string_int(json_map_string_int *map);
+
+json_map_string_int *make_json_map_string_int(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_string_int(void *ctx, json_map_string_int *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_string_int(json_map_string_int *map, const char *key, int val);
+
+typedef struct {
+    char **keys;
+    bool *values;
+    size_t len;
+} json_map_string_bool;
+
+void free_json_map_string_bool(json_map_string_bool *map);
+
+json_map_string_bool *make_json_map_string_bool(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_string_bool(void *ctx, json_map_string_bool *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_string_bool(json_map_string_bool *map, const char *key, bool val);
+
+typedef struct {
+    char **keys;
+    char **values;
+    size_t len;
+} json_map_string_string;
+
+void free_json_map_string_string(json_map_string_string *map);
+
+json_map_string_string *make_json_map_string_string(yajl_val src, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_json_map_string_string(void *ctx, json_map_string_string *map, struct parser_context *ptx, parser_error *err);
+
+int append_json_map_string_string(json_map_string_string *map, const char *key, const char *val);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file
diff --git a/src/lxc/json/logger_json_file.c b/src/lxc/json/logger_json_file.c
new file mode 100644
index 0000000..6abeef4
--- /dev/null
+++ b/src/lxc/json/logger_json_file.c
@@ -0,0 +1,246 @@
+// Generated from json-file.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "logger_json_file.h"
+
+logger_json_file *make_logger_json_file(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ret = NULL;
+	*err = 0;
+	if (tree == NULL)
+		return ret;
+	ret = safe_malloc(sizeof(*ret));
+	{
+		yajl_val tmp = get_val(tree, "log", yajl_t_string);
+		if (tmp != NULL) {
+			char *str = YAJL_GET_STRING(tmp);
+			ret->log = (uint8_t *)safe_strdup(str ? str : "");
+			ret->log_len = str != NULL ? strlen(str) : 0;
+		}
+	}
+	{
+		yajl_val val = get_val(tree, "stream", yajl_t_string);
+		if (val != NULL) {
+			char *str = YAJL_GET_STRING(val);
+			ret->stream = safe_strdup(str ? str : "");
+		}
+	}
+	{
+		yajl_val val = get_val(tree, "time", yajl_t_string);
+		if (val != NULL) {
+			char *str = YAJL_GET_STRING(val);
+			ret->time = safe_strdup(str ? str : "");
+		}
+	}
+	{
+		yajl_val tmp = get_val(tree, "attrs", yajl_t_string);
+		if (tmp != NULL) {
+			char *str = YAJL_GET_STRING(tmp);
+			ret->attrs = (uint8_t *)safe_strdup(str ? str : "");
+			ret->attrs_len = str != NULL ? strlen(str) : 0;
+		}
+	}
+
+	if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+		int i;
+		for (i = 0; i < tree->u.object.len; i++)
+			if (strcmp(tree->u.object.keys[i], "log") &&
+			                strcmp(tree->u.object.keys[i], "stream") &&
+			                strcmp(tree->u.object.keys[i], "time") &&
+			                strcmp(tree->u.object.keys[i], "attrs")) {
+				if (ctx->stderr > 0)
+					fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+			}
+	}
+	return ret;
+}
+
+void free_logger_json_file(logger_json_file *ptr) {
+	if (ptr == NULL)
+		return;
+	free(ptr->log);
+	ptr->log = NULL;
+	free(ptr->stream);
+	ptr->stream = NULL;
+	free(ptr->time);
+	ptr->time = NULL;
+	free(ptr->attrs);
+	ptr->attrs = NULL;
+	free(ptr);
+}
+
+yajl_gen_status gen_logger_json_file(yajl_gen g, logger_json_file *ptr, struct parser_context *ctx, parser_error *err) {
+	yajl_gen_status stat = yajl_gen_status_ok;
+	*err = 0;
+	stat = reformat_start_map(g);
+	if (yajl_gen_status_ok != stat)
+		GEN_SET_ERROR_AND_RETURN(stat, err);
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->log != NULL && ptr->log_len)) {
+		const char *str = "";
+		size_t len = 0;
+		stat = reformat_map_key(g, "log", strlen("log"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->log != NULL) {
+			str = (const char *)ptr->log;
+			len = ptr->log_len;
+		}
+		stat = reformat_string(g, str, len);
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->stream != NULL)) {
+		char *str = "";
+		stat = reformat_map_key(g, "stream", strlen("stream"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->stream != NULL) {
+			str = ptr->stream;
+		}
+		stat = reformat_string(g, str, strlen(str));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->time != NULL)) {
+		char *str = "";
+		stat = reformat_map_key(g, "time", strlen("time"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->time != NULL) {
+			str = ptr->time;
+		}
+		stat = reformat_string(g, str, strlen(str));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr != NULL && ptr->attrs != NULL && ptr->attrs_len)) {
+		const char *str = "";
+		size_t len = 0;
+		stat = reformat_map_key(g, "attrs", strlen("attrs"));
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+		if (ptr != NULL && ptr->attrs != NULL) {
+			str = (const char *)ptr->attrs;
+			len = ptr->attrs_len;
+		}
+		stat = reformat_string(g, str, len);
+		if (yajl_gen_status_ok != stat)
+			GEN_SET_ERROR_AND_RETURN(stat, err);
+	}
+	stat = reformat_end_map(g);
+	if (yajl_gen_status_ok != stat)
+		GEN_SET_ERROR_AND_RETURN(stat, err);
+	return yajl_gen_status_ok;
+}
+
+
+logger_json_file *logger_json_file_parse_file(const char *filename, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ptr = NULL;
+	size_t filesize;
+	char *content = NULL;
+
+	if (filename == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	content = read_file(filename, &filesize);
+	if (content == NULL) {
+		if (asprintf(err, "cannot read the file: %s", filename) < 0)
+			*err = safe_strdup("error allocating memory");
+		return NULL;
+	}
+	ptr = logger_json_file_parse_data(content, ctx, err);
+	free(content);
+	return ptr;
+}
+
+logger_json_file *logger_json_file_parse_file_stream(FILE *stream, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ptr = NULL;
+	size_t filesize;
+	char *content = NULL ;
+
+	if (stream == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	content = fread_file(stream, &filesize);
+	if (content == NULL) {
+		*err = safe_strdup("cannot read the file");
+		return NULL;
+	}
+	ptr = logger_json_file_parse_data(content, ctx, err);
+	free(content);
+	return ptr;
+}
+
+logger_json_file *logger_json_file_parse_data(const char *jsondata, struct parser_context *ctx, parser_error *err) {
+	logger_json_file *ptr  = NULL;
+	yajl_val tree;
+	char errbuf[1024];
+	struct parser_context tmp_ctx;
+
+	if (jsondata == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	if (ctx == NULL) {
+		ctx = &tmp_ctx;
+		memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+	}
+	tree = yajl_tree_parse(jsondata, errbuf, sizeof(errbuf));
+	if (tree == NULL) {
+		if (asprintf(err, "cannot parse the data: %s", errbuf) < 0)
+			*err = safe_strdup("error allocating memory");
+		return NULL;
+	}
+	ptr = make_logger_json_file(tree, ctx, err);
+	yajl_tree_free(tree);
+	return ptr;
+}
+char *logger_json_file_generate_json(logger_json_file *ptr, struct parser_context *ctx, parser_error *err) {
+	yajl_gen g = NULL;
+	struct parser_context tmp_ctx;
+	const unsigned char *gen_buf = NULL;
+	char *json_buf = NULL;
+	size_t gen_len = 0;
+
+	if (ptr == NULL || err == NULL)
+		return NULL;
+
+	*err = NULL;
+	if (ctx == NULL) {
+		ctx = &tmp_ctx;
+		memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+	}
+
+	if (!json_gen_init(&g, ctx)) {
+		*err = safe_strdup("Json_gen init failed");
+		goto out;
+	}
+	if (yajl_gen_status_ok != gen_logger_json_file(g, ptr, ctx, err)) {
+		if (*err == NULL)
+			*err = safe_strdup("Failed to generate json");
+		goto free_out;
+	}
+	yajl_gen_get_buf(g, &gen_buf, &gen_len);
+	if (gen_buf == NULL) {
+		*err = safe_strdup("Error to get generated json");
+		goto free_out;
+	}
+
+	if (gen_len == SIZE_MAX) {
+		*err = safe_strdup("Invalid buffer length");
+		goto free_out;
+	}
+	json_buf = safe_malloc(gen_len + 1);
+	(void)memcpy(json_buf, gen_buf, gen_len);
+	json_buf[gen_len] = '\0';
+
+free_out:
+	yajl_gen_clear(g);
+	yajl_gen_free(g);
+out:
+	return json_buf;
+}
diff --git a/src/lxc/json/logger_json_file.h b/src/lxc/json/logger_json_file.h
new file mode 100644
index 0000000..ad5af7b
--- /dev/null
+++ b/src/lxc/json/logger_json_file.h
@@ -0,0 +1,45 @@
+// Generated from json-file.json. Do not edit!
+#ifndef LOGGER_JSON_FILE_SCHEMA_H
+#define LOGGER_JSON_FILE_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    uint8_t *log;
+    size_t log_len;
+
+    char *stream;
+
+    char *time;
+
+    uint8_t *attrs;
+    size_t attrs_len;
+
+}
+logger_json_file;
+
+void free_logger_json_file(logger_json_file *ptr);
+
+logger_json_file *make_logger_json_file(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_logger_json_file(yajl_gen g, logger_json_file *ptr, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_file(const char *filename, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_file_stream(FILE *stream, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_data(const char *jsondata, struct parser_context *ctx, parser_error *err);
+
+char *logger_json_file_generate_json(logger_json_file *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/json/oci_runtime_hooks.c b/src/lxc/json/oci_runtime_hooks.c
new file mode 100644
index 0000000..41ddb67
--- /dev/null
+++ b/src/lxc/json/oci_runtime_hooks.c
@@ -0,0 +1,52 @@
+/******************************************************************************
+ * Copyright (C), 1988-1999, Huawei Tech. Co., Ltd.
+ * FileName: oci_runtime_hooks.c
+ * Author: maoweiyong   Version: 0.1   Date: 2018-11-07
+ * Explanation: provide oci runtime hooks functions
+ ******************************************************************************/
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <read-file.h>
+#include "oci_runtime_hooks.h"
+
+#include "log.h"
+#include "utils.h"
+
+#define PARSE_ERR_BUFFER_SIZE 1024
+
+oci_runtime_spec_hooks *oci_runtime_spec_hooks_parse_file(const char *filename,
+                                                          struct parser_context *ctx, parser_error *err)
+{
+    yajl_val tree;
+    size_t filesize;
+
+    if (!filename || !err) {
+        return NULL;
+    }
+    *err = NULL;
+    struct parser_context tmp_ctx;
+    if (!ctx) {
+        ctx = &tmp_ctx;
+        memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+    }
+    char *content = read_file(filename, &filesize);
+    char errbuf[PARSE_ERR_BUFFER_SIZE];
+    if (content == NULL) {
+        if (asprintf(err, "cannot read the file: %s", filename) < 0) {
+            *err = safe_strdup("error allocating memory");
+        }
+        return NULL;
+    }
+    tree = yajl_tree_parse(content, errbuf, sizeof(errbuf));
+    free(content);
+    if (tree == NULL) {
+        if (asprintf(err, "cannot parse the file: %s", errbuf) < 0) {
+            *err = safe_strdup("error allocating memory");
+        }
+        return NULL;
+    }
+    oci_runtime_spec_hooks *ptr = make_oci_runtime_spec_hooks(tree, ctx, err);
+    yajl_tree_free(tree);
+    return ptr;
+}
diff --git a/src/lxc/json/oci_runtime_hooks.h b/src/lxc/json/oci_runtime_hooks.h
new file mode 100644
index 0000000..bf570c9
--- /dev/null
+++ b/src/lxc/json/oci_runtime_hooks.h
@@ -0,0 +1,15 @@
+/******************************************************************************
+ * Copyright (C), 1988-1999, Huawei Tech. Co., Ltd.
+ * FileName: oci_runtime_hooks.h
+ * Author: tanyifeng  Version: 0.1   Date: 2018-11-08
+ * Explanation: provide container oci runtime hooks function definition
+ ******************************************************************************/
+#ifndef _CONTAINER_HOOKS_H
+# define _CONTAINER_HOOKS_H
+
+# include "oci_runtime_spec.h"
+
+oci_runtime_spec_hooks *oci_runtime_spec_hooks_parse_file(const char *filename,
+                                                          struct parser_context *ctx, parser_error *err);
+
+#endif
diff --git a/src/lxc/json/oci_runtime_spec.c b/src/lxc/json/oci_runtime_spec.c
new file mode 100644
index 0000000..fd342de
--- /dev/null
+++ b/src/lxc/json/oci_runtime_spec.c
@@ -0,0 +1,195 @@
+// Generated from spec.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "oci_runtime_spec.h"
+
+oci_runtime_spec_hooks *make_oci_runtime_spec_hooks(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+    oci_runtime_spec_hooks *ret = NULL;
+    *err = 0;
+    if (tree == NULL)
+        return ret;
+    ret = safe_malloc(sizeof(*ret));
+    {
+        yajl_val tmp = get_val(tree, "prestart", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->prestart_len = YAJL_GET_ARRAY(tmp)->len;
+            ret->prestart = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->prestart));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                ret->prestart[i] = make_defs_hook(val, ctx, err);
+                if (ret->prestart[i] == NULL) {
+                    free_oci_runtime_spec_hooks(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "poststart", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->poststart_len = YAJL_GET_ARRAY(tmp)->len;
+            ret->poststart = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->poststart));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                ret->poststart[i] = make_defs_hook(val, ctx, err);
+                if (ret->poststart[i] == NULL) {
+                    free_oci_runtime_spec_hooks(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "poststop", yajl_t_array);
+        if (tmp != NULL && YAJL_GET_ARRAY(tmp) != NULL && YAJL_GET_ARRAY(tmp)->len > 0) {
+            size_t i;
+            ret->poststop_len = YAJL_GET_ARRAY(tmp)->len;
+            ret->poststop = safe_malloc((YAJL_GET_ARRAY(tmp)->len + 1) * sizeof(*ret->poststop));
+            for (i = 0; i < YAJL_GET_ARRAY(tmp)->len; i++) {
+                yajl_val val = YAJL_GET_ARRAY(tmp)->values[i];
+                ret->poststop[i] = make_defs_hook(val, ctx, err);
+                if (ret->poststop[i] == NULL) {
+                    free_oci_runtime_spec_hooks(ret);
+                    return NULL;
+                }
+            }
+        }
+    }
+
+    if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+        int i;
+        for (i = 0; i < tree->u.object.len; i++)
+            if (strcmp(tree->u.object.keys[i], "prestart") &&
+                strcmp(tree->u.object.keys[i], "poststart") &&
+                strcmp(tree->u.object.keys[i], "poststop")) {
+                if (ctx->stderr > 0)
+                    fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+            }
+        }
+    return ret;
+}
+
+void free_oci_runtime_spec_hooks(oci_runtime_spec_hooks *ptr) {
+    if (ptr == NULL)
+        return;
+    if (ptr->prestart != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->prestart_len; i++)
+            if (ptr->prestart[i] != NULL) {
+                free_defs_hook(ptr->prestart[i]);
+                ptr->prestart[i] = NULL;
+            }
+        free(ptr->prestart);
+        ptr->prestart = NULL;
+    }
+    if (ptr->poststart != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->poststart_len; i++)
+            if (ptr->poststart[i] != NULL) {
+                free_defs_hook(ptr->poststart[i]);
+                ptr->poststart[i] = NULL;
+            }
+        free(ptr->poststart);
+        ptr->poststart = NULL;
+    }
+    if (ptr->poststop != NULL) {
+        size_t i;
+        for (i = 0; i < ptr->poststop_len; i++)
+            if (ptr->poststop[i] != NULL) {
+                free_defs_hook(ptr->poststop[i]);
+                ptr->poststop[i] = NULL;
+            }
+        free(ptr->poststop);
+        ptr->poststop = NULL;
+    }
+    free(ptr);
+}
+
+yajl_gen_status gen_oci_runtime_spec_hooks(yajl_gen g, oci_runtime_spec_hooks *ptr, struct parser_context *ctx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    *err = 0;
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->prestart != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "prestart", strlen("prestart"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->prestart != NULL) {
+            len = ptr->prestart_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = gen_defs_hook(g, ptr->prestart[i], ctx, err);
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->poststart != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "poststart", strlen("poststart"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->poststart != NULL) {
+            len = ptr->poststart_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = gen_defs_hook(g, ptr->poststart[i], ctx, err);
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr != NULL && ptr->poststop != NULL)) {
+        size_t len = 0, i;
+        stat = reformat_map_key(g, "poststop", strlen("poststop"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr != NULL && ptr->poststop != NULL) {
+            len = ptr->poststop_len;
+        }
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 0);
+        stat = reformat_start_array(g);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        for (i = 0; i < len; i++) {
+            stat = gen_defs_hook(g, ptr->poststop[i], ctx, err);
+            if (yajl_gen_status_ok != stat)
+                GEN_SET_ERROR_AND_RETURN(stat, err);
+        }
+        stat = reformat_end_array(g);
+        if (!len && !(ctx->options & GEN_OPTIONS_SIMPLIFY))
+            yajl_gen_config(g, yajl_gen_beautify, 1);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    return yajl_gen_status_ok;
+}
diff --git a/src/lxc/json/oci_runtime_spec.h b/src/lxc/json/oci_runtime_spec.h
new file mode 100644
index 0000000..ef3f161
--- /dev/null
+++ b/src/lxc/json/oci_runtime_spec.h
@@ -0,0 +1,37 @@
+// Generated from spec.json. Do not edit!
+#ifndef OCI_RUNTIME_SPEC_SCHEMA_H
+#define OCI_RUNTIME_SPEC_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+#include "defs.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    defs_hook **prestart;
+    size_t prestart_len;
+
+    defs_hook **poststart;
+    size_t poststart_len;
+
+    defs_hook **poststop;
+    size_t poststop_len;
+
+}
+oci_runtime_spec_hooks;
+
+void free_oci_runtime_spec_hooks(oci_runtime_spec_hooks *ptr);
+
+oci_runtime_spec_hooks *make_oci_runtime_spec_hooks(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_oci_runtime_spec_hooks(yajl_gen g, oci_runtime_spec_hooks *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/json/read-file.c b/src/lxc/json/read-file.c
new file mode 100644
index 0000000..34ebeed
--- /dev/null
+++ b/src/lxc/json/read-file.c
@@ -0,0 +1,95 @@
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#include <config.h>
+#include "read-file.h"
+
+#ifndef O_CLOEXEC
+#define O_CLOEXEC 02000000
+#endif
+
+char *fread_file(FILE *stream, size_t *length)
+{
+	char *buf = NULL, *tmpbuf = NULL;
+	size_t off = 0;
+
+	while (1) {
+		size_t ret, newsize;
+
+		newsize = off + BUFSIZ + 1;
+		tmpbuf = (char *)calloc(1, newsize);
+		if (tmpbuf == NULL) {
+			goto out;
+		}
+
+		if (buf) {
+			memcpy(tmpbuf, buf, off);
+
+			memset(buf, 0, off);
+
+			free(buf);
+		}
+
+		buf = tmpbuf;
+		ret = fread(buf + off, 1, BUFSIZ, stream);
+		if (!ret && ferror(stream)) {
+			tmpbuf = NULL;
+			goto out;
+		}
+		if (ret < BUFSIZ || feof(stream)) {
+			*length = off + ret + 1;
+			buf[*length - 1] = '\0';
+			return buf;
+		}
+		off += BUFSIZ;
+	}
+out:
+	if (buf) {
+		free(buf);
+	}
+	if (tmpbuf) {
+		free(tmpbuf);
+	}
+	return NULL;
+
+}
+
+char *read_file(const char *path, size_t *length)
+{
+	char *buf = NULL;
+	char rpath[PATH_MAX + 1] = {0};
+	int fd = -1;
+	int tmperrno;
+	FILE *fp = NULL;
+
+	if (!path || !length) {
+		return NULL;
+	}
+
+	if (strlen(path) > PATH_MAX || NULL == realpath(path, rpath)) {
+		return NULL;
+	}
+
+	fd = open(rpath, O_RDONLY | O_CLOEXEC);
+	if (fd < 0) {
+		return NULL;
+	}
+
+	fp = fdopen(fd, "r");
+	tmperrno = errno;
+	if (!fp) {
+		close(fd);
+		errno = tmperrno;
+		return NULL;
+	}
+
+	buf = fread_file(fp, length);
+	fclose(fp);
+	return buf;
+}
diff --git a/src/lxc/json/read-file.h b/src/lxc/json/read-file.h
new file mode 100644
index 0000000..5d6e0eb
--- /dev/null
+++ b/src/lxc/json/read-file.h
@@ -0,0 +1,11 @@
+#ifndef READ_FILE_H
+#define READ_FILE_H
+
+#include <stddef.h>
+#include <stdio.h>
+
+extern char *fread_file(FILE *stream, size_t *length);
+
+extern char *read_file(const char *path, size_t *length);
+
+#endif
diff --git a/src/lxc/meson.build b/src/lxc/meson.build
index f33257c..3166401 100644
--- a/src/lxc/meson.build
+++ b/src/lxc/meson.build
@@ -25,7 +25,6 @@ liblxcfs_version_file = configure_file(
 )
 
 liblxc_sources = files(
-    'cgroups/cgfsng.c',
     'cgroups/cgroup.c',
     'cgroups/cgroup.h',
     'cgroups/cgroup2_devices.c',
@@ -139,6 +138,31 @@ liblxc_sources = files(
     'uuid.c',
     'uuid.h')
 
+if want_isulad
+    liblxc_sources += files(
+        'cgroups/isulad_cgfsng.c',
+        'exec_commands.c',
+        'exec_commands.h',
+        'isulad_utils.c',
+        'isulad_utils.h',
+        'path.c',
+        'path.h',
+        'json/defs.c',
+        'json/defs.h',
+        'json/json_common.c',
+        'json/json_common.h',
+        'json/logger_json_file.c',
+        'json/logger_json_file.h',
+        'json/oci_runtime_hooks.c',
+        'json/oci_runtime_hooks.h',
+        'json/oci_runtime_spec.c',
+        'json/oci_runtime_spec.h',
+        'json/read-file.c',
+        'json/read-file.h')
+else
+    liblxc_sources += files('cgroups/cgfsng.c')
+endif
+
 if want_apparmor and libapparmor.found()
     liblxc_sources += files('lsm/apparmor.c')
 endif
diff --git a/src/lxc/path.c b/src/lxc/path.c
new file mode 100644
index 0000000..3881058
--- /dev/null
+++ b/src/lxc/path.c
@@ -0,0 +1,521 @@
+/* SPDX-License-Identifier: LGPL-2.1+ */
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. Allrights reserved
+ * Description: isulad utils
+ * Author: lifeng
+ * Create: 2020-04-11
+******************************************************************************/
+#include <unistd.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/param.h>
+#include <libgen.h>
+
+#include "path.h"
+#include "log.h"
+#include "isulad_utils.h"
+
+lxc_log_define(lxc_path_ui, lxc);
+
+#define ISSLASH(C) ((C) == '/')
+#define IS_ABSOLUTE_FILE_NAME(F) (ISSLASH ((F)[0]))
+
+static bool do_clean_path_continue(const char *endpos, const char *stpos, const char *respath, char **dst)
+{
+	if (endpos - stpos == 1 && stpos[0] == '.') {
+		return true;
+	} else if (endpos - stpos == 2 && stpos[0] == '.' && stpos[1] == '.') {
+		char *dest = *dst;
+		if (dest <= respath + 1) {
+			return true;
+		}
+		for (--dest; dest > respath && !ISSLASH(dest[-1]); --dest) {
+			*dst = dest;
+			return true;
+		}
+		*dst = dest;
+		return true;
+	}
+	return false;
+}
+
+int do_clean_path(const char *respath, const char *limit_respath,
+                  const char *stpos, char **dst)
+{
+	char *dest = *dst;
+	const char *endpos = NULL;
+
+	for (endpos = stpos; *stpos; stpos = endpos) {
+		while (ISSLASH(*stpos)) {
+			++stpos;
+		}
+
+		for (endpos = stpos; *endpos && !ISSLASH(*endpos); ++endpos) {
+		}
+
+		if (endpos - stpos == 0) {
+			break;
+		} else if (do_clean_path_continue(endpos, stpos, respath, &dest)) {
+			continue;
+		}
+
+		if (!ISSLASH(dest[-1])) {
+			*dest++ = '/';
+		}
+
+		if (dest + (endpos - stpos) >= limit_respath) {
+			ERROR("Path is too long");
+			if (dest > respath + 1) {
+				dest--;
+			}
+			*dest = '\0';
+			return -1;
+		}
+
+		memcpy(dest, stpos, (size_t)(endpos - stpos));
+		dest += endpos - stpos;
+		*dest = '\0';
+	}
+	*dst = dest;
+	return 0;
+}
+
+char *cleanpath(const char *path, char *realpath, size_t realpath_len)
+{
+	char *respath = NULL;
+	char *dest = NULL;
+	const char *stpos = NULL;
+	const char *limit_respath = NULL;
+
+	if (path == NULL || path[0] == '\0' || \
+	                realpath == NULL || (realpath_len < PATH_MAX)) {
+		return NULL;
+	}
+
+	respath = realpath;
+
+	memset(respath, 0, realpath_len);
+	limit_respath = respath + PATH_MAX;
+
+	if (!IS_ABSOLUTE_FILE_NAME(path)) {
+		if (!getcwd(respath, PATH_MAX)) {
+			ERROR("Failed to getcwd");
+			respath[0] = '\0';
+			goto error;
+		}
+		dest = strchr(respath, '\0');
+		if (dest == NULL) {
+			ERROR("Failed to get the end of respath");
+			goto error;
+		}
+		if (strlen(path) > (PATH_MAX - strlen(respath) - 1)) {
+			ERROR("Path is too long");
+			goto error;
+		}
+		strcat(respath, path);
+		stpos = path;
+	} else {
+		dest = respath;
+		*dest++ = '/';
+		stpos = path;
+	}
+
+	if (do_clean_path(respath, limit_respath, stpos, &dest)) {
+		goto error;
+	}
+
+	if (dest > respath + 1 && ISSLASH(dest[-1])) {
+		--dest;
+	}
+	*dest = '\0';
+
+	return respath;
+
+error:
+	return NULL;
+}
+
+static int do_path_realloc(const char *start, const char *end,
+                           char **rpath, char **dest, const char **rpath_limit)
+{
+	long long dest_offset = *dest - *rpath;
+	char *new_rpath = NULL;
+	size_t new_size;
+	int nret = 0;
+	size_t gap = 0;
+
+	if (*dest + (end - start) < *rpath_limit) {
+		return 0;
+	}
+
+	gap = (size_t)(end - start) + 1;
+	new_size = (size_t)(*rpath_limit - *rpath);
+	if (new_size > SIZE_MAX - gap) {
+		ERROR("Out of range!");
+		return -1;
+	}
+
+	if (gap > PATH_MAX) {
+		new_size += gap;
+	} else {
+		new_size += PATH_MAX;
+	}
+	nret = lxc_mem_realloc((void **)&new_rpath, new_size, *rpath, PATH_MAX);
+	if (nret) {
+		ERROR("Failed to realloc memory for files limit variables");
+		return -1;
+	}
+	*rpath = new_rpath;
+	*rpath_limit = *rpath + new_size;
+
+	*dest = *rpath + dest_offset;
+
+	return 0;
+}
+
+static int do_get_symlinks_copy_buf(const char *buf, const char *prefix, size_t prefix_len,
+                                    char **rpath, char **dest)
+{
+	if (IS_ABSOLUTE_FILE_NAME(buf)) {
+		if (prefix_len) {
+			memcpy(*rpath, prefix, prefix_len);
+		}
+		*dest = *rpath + prefix_len;
+		*(*dest)++ = '/';
+	} else {
+		if (*dest > *rpath + prefix_len + 1) {
+			for (--(*dest); *dest > *rpath && !ISSLASH((*dest)[-1]); --(*dest)) {
+				continue;
+			}
+		}
+	}
+	return 0;
+}
+
+static int do_get_symlinks(const char **fullpath, const char *prefix, size_t prefix_len,
+                           char **rpath, char **dest, const char **end,
+                           int *num_links, char **extra_buf)
+{
+	char *buf = NULL;
+	size_t len;
+	ssize_t n;
+	int ret = -1;
+
+	if (++(*num_links) > MAXSYMLINKS) {
+		ERROR("Too many links in '%s'", *fullpath);
+		goto out;
+	}
+
+	buf = lxc_common_calloc_s(PATH_MAX);
+	if (buf == NULL) {
+		ERROR("Out of memory");
+		goto out;
+	}
+
+	n = readlink(*rpath, buf, PATH_MAX - 1);
+	if (n < 0) {
+		goto out;
+	}
+	buf[n] = '\0';
+
+	if (*extra_buf == NULL) {
+		*extra_buf = lxc_common_calloc_s(PATH_MAX);
+		if (*extra_buf == NULL) {
+			ERROR("Out of memory");
+			goto out;
+		}
+	}
+
+	len = strlen(*end);
+	if (len >= (size_t)(PATH_MAX - n)) {
+		ERROR("Path is too long");
+		goto out;
+	}
+
+	memmove(&(*extra_buf)[n], *end, len + 1);
+	memcpy(*extra_buf, buf, (size_t)n);
+
+	*fullpath = *end = *extra_buf;
+
+	if (do_get_symlinks_copy_buf(buf, prefix, prefix_len, rpath, dest) != 0) {
+		goto out;
+	}
+
+	ret = 0;
+out:
+	free(buf);
+	return ret;
+}
+
+static bool do_eval_symlinks_in_scope_is_symlink(const char *path)
+{
+	struct stat st;
+
+	if (lstat(path, &st) < 0) {
+		return true;
+	}
+
+	if (!S_ISLNK(st.st_mode)) {
+		return true;
+	}
+	return false;
+}
+
+static void do_eval_symlinks_skip_slash(const char **start, const char **end)
+{
+	while (ISSLASH(**start)) {
+		++(*start);
+	}
+
+	for (*end = *start; **end && !ISSLASH(**end); ++(*end)) {
+	}
+}
+
+static inline void skip_dest_traling_slash(char **dest, char **rpath, size_t prefix_len)
+{
+	if (*dest > *rpath + prefix_len + 1) {
+		for (--(*dest); *dest > *rpath && !ISSLASH((*dest)[-1]); --(*dest)) {
+			continue;
+		}
+	}
+}
+
+static inline bool is_current_char(const char c)
+{
+	return c == '.';
+}
+
+static inline bool is_specify_current(const char *end, const char *start)
+{
+	return (end - start == 1) && is_current_char(start[0]);
+}
+
+static inline bool is_specify_parent(const char *end, const char *start)
+{
+	return (end - start == 2) && is_current_char(start[0]) && is_current_char(start[1]);
+}
+
+static int do_eval_symlinks_in_scope(const char *fullpath, const char *prefix,
+                                     size_t prefix_len,
+                                     char **rpath, char **dest, const char *rpath_limit)
+{
+	const char *start = NULL;
+	const char *end = NULL;
+	char *extra_buf = NULL;
+	int nret = 0;
+	int num_links = 0;
+
+	start = fullpath + prefix_len;
+	for (end = start; *start; start = end) {
+		do_eval_symlinks_skip_slash(&start, &end);
+		if (end - start == 0) {
+			break;
+		} else if (is_specify_current(end, start)) {
+			;
+		} else if (is_specify_parent(end, start)) {
+			skip_dest_traling_slash(dest, rpath, prefix_len);
+		} else {
+			if (!ISSLASH((*dest)[-1])) {
+				*(*dest)++ = '/';
+			}
+
+			nret = do_path_realloc(start, end, rpath, dest, &rpath_limit);
+			if (nret != 0) {
+				nret = -1;
+				goto out;
+			}
+
+			memcpy(*dest, start, (size_t)(end - start));
+			*dest += end - start;
+			**dest = '\0';
+
+			if (do_eval_symlinks_in_scope_is_symlink(*rpath)) {
+				continue;
+			}
+
+			nret = do_get_symlinks(&fullpath, prefix, prefix_len, rpath, dest, &end, &num_links, &extra_buf);
+			if (nret != 0) {
+				nret = -1;
+				goto out;
+			}
+		}
+	}
+out:
+	free(extra_buf);
+	return nret;
+}
+
+static char *eval_symlinks_in_scope(const char *fullpath, const char *rootpath)
+{
+	char resroot[PATH_MAX] = {0};
+	char *root = NULL;
+	char *rpath = NULL;
+	char *dest = NULL;
+	char *prefix = NULL;
+	const char *rpath_limit = NULL;
+	size_t prefix_len;
+
+	if (fullpath == NULL || rootpath == NULL) {
+		return NULL;
+	}
+
+	root = cleanpath(rootpath, resroot, sizeof(resroot));
+	if (root == NULL) {
+		ERROR("Failed to get cleaned path");
+		return NULL;
+	}
+
+	if (!strcmp(fullpath, root)) {
+		return safe_strdup(fullpath);
+	}
+
+	if (strstr(fullpath, root) == NULL) {
+		ERROR("Path '%s' is not in '%s'", fullpath, root);
+		return NULL;
+	}
+
+	rpath = lxc_common_calloc_s(PATH_MAX);
+	if (rpath == NULL) {
+		ERROR("Out of memory");
+		goto out;
+	}
+	rpath_limit = rpath + PATH_MAX;
+
+	prefix = root;
+	prefix_len = (size_t)strlen(prefix);
+	if (!strcmp(prefix, "/")) {
+		prefix_len = 0;
+	}
+
+	dest = rpath;
+	if (prefix_len) {
+		memcpy(rpath, prefix, prefix_len);
+		dest += prefix_len;
+	}
+	*dest++ = '/';
+
+	if (do_eval_symlinks_in_scope(fullpath, prefix, prefix_len, &rpath, &dest,
+	                              rpath_limit)) {
+		goto out;
+	}
+
+	if (dest > rpath + prefix_len + 1 && ISSLASH(dest[-1])) {
+		--dest;
+	}
+	*dest = '\0';
+	return rpath;
+
+out:
+	free(rpath);
+	return NULL;
+}
+
+// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an
+// absolute path. This function handles paths in a platform-agnostic manner.
+char *follow_symlink_in_scope(const char *fullpath, const char *rootpath)
+{
+	char resfull[PATH_MAX] = {0}, *full = NULL;
+	char resroot[PATH_MAX] = {0}, *root = NULL;
+
+	full = cleanpath(fullpath, resfull, PATH_MAX);
+	if (!full) {
+		ERROR("Failed to get cleaned path");
+		return NULL;
+	}
+
+	root = cleanpath(rootpath, resroot, PATH_MAX);
+	if (!root) {
+		ERROR("Failed to get cleaned path");
+		return NULL;
+	}
+
+	return eval_symlinks_in_scope(full, root);
+}
+
+// Rel returns a relative path that is lexically equivalent to targpath when
+// joined to basepath with an intervening separator. That is,
+// Join(basepath, Rel(basepath, targpath)) is equivalent to targpath itself.
+// On success, the returned path will always be relative to basepath,
+// even if basepath and targpath share no elements.
+// An error is returned if targpath can't be made relative to basepath or if
+// knowing the current working directory would be necessary to compute it.
+// Rel calls Clean on the result.
+char *path_relative(const char *basepath, const char *targpath)
+{
+	char resbase[PATH_MAX] = {0}, *base = NULL;
+	char restarg[PATH_MAX] = {0}, *targ = NULL;
+	size_t bl = 0, tl = 0, b0 = 0, bi = 0, t0 = 0, ti = 0;
+
+	base = cleanpath(basepath, resbase, PATH_MAX);
+	if (!base) {
+		ERROR("Failed to get cleaned path");
+		return NULL;
+	}
+
+	targ = cleanpath(targpath, restarg, PATH_MAX);
+	if (!targ) {
+		ERROR("Failed to get cleaned path");
+		return NULL;
+	}
+
+	if (strcmp(base, targ) == 0)
+		return safe_strdup(".");
+
+	bl = strlen(base);
+	tl = strlen(targ);
+	while(true) {
+		while(bi < bl && !ISSLASH(base[bi]))
+			bi++;
+		while(ti < tl && !ISSLASH(targ[ti]))
+			ti++;
+		//not the same string
+		if (((bi - b0) != (ti - t0)) || strncmp(base + b0, targ + t0, bi - b0))
+			break;
+		if (bi < bl)
+			bi++;
+		if (ti < tl)
+			ti++;
+		b0 = bi;
+		t0 = ti;
+	}
+
+	if (b0 != bl) {
+		// Base elements left. Must go up before going down.
+		size_t seps = 0, i;
+		size_t ncopyed = 0, seps_size;
+		char *buf = NULL;
+
+		for (bi = b0; bi < bl; bi++) {
+			if (ISSLASH(base[bi]))
+				seps++;
+		}
+		//strlen(..) + strlen(/..) + '\0'
+		seps_size = 2 + seps * 3 + 1;
+		if (t0 != tl)
+			seps_size += 1 + tl - t0;
+
+		buf = calloc(seps_size, 1);
+		if (!buf) {
+			ERROR("Out of memory");
+			return NULL;
+		}
+		buf[ncopyed++] = '.';
+		buf[ncopyed++] = '.';
+		for (i = 0; i < seps; i++) {
+			buf[ncopyed++] = '/';
+			buf[ncopyed++] = '.';
+			buf[ncopyed++] = '.';
+		}
+		if (t0 != tl) {
+			buf[ncopyed++] = '/';
+			memcpy(buf + ncopyed, targ + t0, tl - t0 + 1);
+		}
+		return buf;
+	}
+
+	return safe_strdup(targ + t0);
+}
diff --git a/src/lxc/path.h b/src/lxc/path.h
new file mode 100644
index 0000000..59bbc59
--- /dev/null
+++ b/src/lxc/path.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: LGPL-2.1+ */
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2020. Allrights reserved
+ * Description: isulad utils
+ * Author: lifeng
+ * Create: 2020-04-11
+******************************************************************************/
+#ifndef __ISULAD_PATH_H_
+#define __ISULAD_PATH_H_
+
+#include <stdbool.h>
+
+/*
+ * cleanpath is similar to realpath of glibc, but not expands symbolic links,
+ * and not check the existence of components of the path.
+ */
+char *cleanpath(const char *path, char *realpath, size_t realpath_len);
+
+// FollowSymlinkInScope is a wrapper around evalSymlinksInScope that returns an
+// absolute path. This function handles paths in a platform-agnostic manner.
+char *follow_symlink_in_scope(const char *fullpath, const char *rootpath);
+
+// Rel returns a relative path that is lexically equivalent to targpath when
+// joined to basepath with an intervening separator. That is,
+// Join(basepath, Rel(basepath, targpath)) is equivalent to targpath itself.
+// On success, the returned path will always be relative to basepath,
+// even if basepath and targpath share no elements.
+// An error is returned if targpath can't be made relative to basepath or if
+// knowing the current working directory would be necessary to compute it.
+// Rel calls Clean on the result.
+char *path_relative(const char *basepath, const char *targpath);
+
+#endif
-- 
2.25.1