client/chunkserver/curve-nbd: support 512 aligned IO requests

opencurve · Jul 14, 2021 · 1e62116 · 1e62116
1 parent 6d3c1be
commit 1e62116
Show file tree

Hide file tree

Showing 57 changed files with 1,876 additions and 121 deletions.
diff --git a/conf/chunkserver.conf.example b/conf/chunkserver.conf.example
@@ -14,6 +14,8 @@ global.chunk_size=16777216
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings

diff --git a/conf/client.conf b/conf/client.conf
@@ -164,3 +164,10 @@ discard.enable=true
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
diff --git a/conf/cs_client.conf b/conf/cs_client.conf
@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
diff --git a/conf/py_client.conf b/conf/py_client.conf
@@ -153,3 +153,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
diff --git a/conf/snap_client.conf b/conf/snap_client.conf
@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
diff --git a/curve-ansible/roles/generate_config/defaults/main.yml b/curve-ansible/roles/generate_config/defaults/main.yml
@@ -160,6 +160,7 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
+chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -239,6 +240,8 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
+client_alignment_common: 512
+client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

diff --git a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2 b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
@@ -14,6 +14,8 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
+# minimum alignment for io request
+global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

diff --git a/curve-ansible/roles/generate_config/templates/client.conf.j2 b/curve-ansible/roles/generate_config/templates/client.conf.j2
@@ -175,3 +175,10 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume={{ client_alignment_common }}
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume={{ client_alignment_clone }}
diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.0 b/deploy/local/chunkserver/conf/chunkserver.conf.0
@@ -28,6 +28,8 @@ global.chunk_size=16777216
 global.meta_page_size=4096
 global.location_limit=3000
 
+global.min_io_alignment=512
+
 #
 # MDS settings
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.1 b/deploy/local/chunkserver/conf/chunkserver.conf.1
@@ -28,6 +28,8 @@ global.chunk_size=16777216
 global.meta_page_size=4096
 global.location_limit=3000
 
+global.min_io_alignment=512
+
 #
 # MDS settings
 #

diff --git a/deploy/local/chunkserver/conf/chunkserver.conf.2 b/deploy/local/chunkserver/conf/chunkserver.conf.2
@@ -28,6 +28,8 @@ global.chunk_size=16777216
 global.meta_page_size=4096
 global.location_limit=3000
 
+global.min_io_alignment=512
+
 #
 # MDS settings
 #

diff --git a/include/chunkserver/chunkserver_common.h b/include/chunkserver/chunkserver_common.h
@@ -123,9 +123,6 @@ inline std::string ToGroupIdString(const LogicPoolID &logicPoolId,
 }
 #define ToGroupIdStr   ToGroupIdString
 
-// TODO(wudmeiao): 是否需要考虑可配置
-const uint32_t kOpRequestAlignSize = 4096;
-
 }  // namespace chunkserver
 }  // namespace curve
 

diff --git a/nbd/src/NBDController.cpp b/nbd/src/NBDController.cpp
@@ -47,7 +47,7 @@ int IOController::InitDevAttr(NBDConfig* config, uint64_t size,
     int ret = -1;
 
     do {
-        ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, CURVE_NBD_BLKSIZE);
+        ret = ioctl(nbdFd_, NBD_SET_BLKSIZE, config->block_size);
         if (ret < 0) {
             break;
         }
@@ -259,7 +259,7 @@ int NetLinkController::SetUp(NBDConfig* config, int sockfd,
     if (index < 0) {
         return index;
     }
-    ret = check_block_size(index, CURVE_NBD_BLKSIZE);
+    ret = check_block_size(index, config->block_size);
     if (ret < 0) {
         return ret;
     }
@@ -393,7 +393,7 @@ int NetLinkController::ConnectInternal(NBDConfig* config, int sockfd,
         NLA_PUT_U64(msg, NBD_ATTR_TIMEOUT, config->timeout);
     }
     NLA_PUT_U64(msg, NBD_ATTR_SIZE_BYTES, size);
-    NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, CURVE_NBD_BLKSIZE);
+    NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, config->block_size);
     NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags);
 
     sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS);

diff --git a/nbd/src/define.h b/nbd/src/define.h
@@ -55,7 +55,6 @@ namespace nbd {
 
 #define HELP_INFO 1
 #define VERSION_INFO 2
-#define CURVE_NBD_BLKSIZE 4096UL    // CURVE后端当前支持4096大小对齐的IO
 
 #define NBD_MAX_PATH "/sys/module/nbd/parameters/nbds_max"
 #define PROCESS_NAME "curve-nbd"
@@ -88,6 +87,8 @@ struct NBDConfig {
     int retry_times = 25;
     // unmap重试之间的睡眠间隔
     int sleep_ms = 200;
+    // device's block size
+    int block_size = 4096;
 };
 
 // 用户命令类型

diff --git a/nbd/src/main.cpp b/nbd/src/main.cpp
@@ -87,6 +87,7 @@ static void Usage() {
         << "  --max_part <limit>      Override for module param max_part\n"
         << "  --timeout <seconds>     Set nbd request timeout\n"
         << "  --try-netlink           Use the nbd netlink interface\n"
+        << "  --block-size            NBD Devices's block size, default is 4096, support 512 and 4096\n"  // NOLINT
         << "Unmap options:\n"
         << "  -f, --force                 Force unmap even if the device is mounted\n"              // NOLINT
         << "  --retry_times <limit>       The number of retries waiting for the process to exit\n"  // NOLINT

diff --git a/nbd/src/util.cpp b/nbd/src/util.cpp
@@ -173,6 +173,17 @@ int parse_args(std::vector<const char*>& args, std::ostream *err_msg,   // NOLIN
                 *err_msg << "curve-nbd: Invalid argument for sleep_ms!";
                 return -EINVAL;
             }
+        } else if (argparse_witharg(args, i, &cfg->block_size, err, "--block_size", (char*)(NULL))) {  // NOLINT
+            if (!err.str().empty()) {
+                *err_msg << "curve-nbd: " << err.str();
+                return -EINVAL;
+            }
+
+            if (cfg->block_size != 512 && cfg->block_size != 4096) {
+                *err_msg << "curve-nbd: Invalid block size, only support 512 "
+                            "or 4096";
+                return -EINVAL;
+            }
         } else {
             ++i;
         }
@@ -322,7 +333,6 @@ int check_size_from_file(const std::string& path, uint64_t expected_size) {
 
     uint64_t size = 0;
     ifs >> size;
-    size *= CURVE_NBD_BLKSIZE;
 
     if (size == 0) {
         // Newer kernel versions will report real size only after nbd

diff --git a/robot/curve_robot.txt b/robot/curve_robot.txt
@@ -1449,11 +1449,11 @@ read write iosize 512
     ${fd}   open libcurve file
     ${new_fd}       Convert To Integer    ${fd}
     ${rc}    write libcurve file error   ${new_fd}    length=512
-    ${expect_rc}    evaluate    int(-22)
-    should be equal    ${rc}   ${expect_rc}
-    ${rc}    read libcurve file   ${new_fd}   length=512
-    ${expect_rc}    evaluate    int(-22)
+    ${expect_rc}    evaluate    int(512)
     should be equal    ${rc}   ${expect_rc}
+    ${content}    read libcurve file   ${new_fd}   length=512
+    ${expect_content}    evaluate    str("a")*512
+    should be equal    ${content}   ${expect_content}
     [Teardown]	    file clean      ${new_fd}
 
 
@@ -1480,11 +1480,11 @@ read write iosize 512
     ${fd}   open libcurve file
     ${new_fd}       Convert To Integer    ${fd}
     ${rc}    write libcurve file error   ${new_fd}    length=512
-    ${expect_rc}    evaluate    int(-22)
-    should be equal    ${rc}   ${expect_rc}
-    ${rc}    read libcurve file   ${new_fd}   length=512
-    ${expect_rc}    evaluate    int(-22)
+    ${expect_rc}    evaluate    int(512)
     should be equal    ${rc}   ${expect_rc}
+    ${content}    read libcurve file   ${new_fd}   length=512
+    ${expect_content}    evaluate    str("a")*512
+    should be equal    ${content}   ${expect_content}
     [Teardown]	    file clean      ${new_fd}
 
 read write iosize 1k
@@ -1493,9 +1493,22 @@ read write iosize 1k
     ${fd}   open libcurve file
     ${new_fd}       Convert To Integer    ${fd}
     ${rc}    write libcurve file error   ${new_fd}    length=1024
+    ${expect_rc}    evaluate    int(1024)
+    should be equal    ${rc}   ${expect_rc}
+    ${content}    read libcurve file   ${new_fd}   length=1024
+    ${expect_content}    evaluate    str("a")*1024
+    should be equal    ${content}   ${expect_content}
+    [Teardown]	    file clean      ${new_fd}
+
+read write iosize not 512 alignment
+    [Tags]    P0    base    first release
+    [Setup]	file init
+    ${fd}   open libcurve file
+    ${new_fd}       Convert To Integer    ${fd}
+    ${rc}    write libcurve file error   ${new_fd}    length=511
     ${expect_rc}    evaluate    int(-22)
     should be equal    ${rc}   ${expect_rc}
-    ${rc}    read libcurve file   ${new_fd}   length=1024
+    ${rc}    read libcurve file    ${new_fd}    length=511
     ${expect_rc}    evaluate    int(-22)
     should be equal    ${rc}   ${expect_rc}
     [Teardown]	    file clean      ${new_fd}
@@ -2552,9 +2565,3 @@ check chunkserver should be exsits
 start mds with sleep
      start mds
 	 Sleep		5s
-
-
-
-
-
-
diff --git a/src/chunkserver/chunk_service.cpp b/src/chunkserver/chunk_service.cpp
@@ -36,6 +36,8 @@
 #include "src/chunkserver/op_request.h"
 #include "src/chunkserver/chunk_service_closure.h"
 
+#include "src/common/fast_align.h"
+
 namespace curve {
 namespace chunkserver {
 
@@ -541,17 +543,8 @@ bool ChunkServiceImpl::CheckRequestOffsetAndLength(uint32_t offset,
         return false;
     }
 
-    // 检查offset是否对齐
-    if (offset % kOpRequestAlignSize != 0) {
-        return false;
-    }
-
-    // 检查len是否对齐
-    if (len % kOpRequestAlignSize != 0) {
-        return false;
-    }
-
-    return true;
+    return common::is_aligned(offset, FLAGS_minIoAlignment) &&
+           common::is_aligned(len, FLAGS_minIoAlignment);
 }
 
 }  // namespace chunkserver

diff --git a/src/chunkserver/chunkserver.cpp b/src/chunkserver/chunkserver.cpp
@@ -106,6 +106,12 @@ int ChunkServer::Run(int argc, char** argv) {
     // ============================初始化各模块==========================//
     LOG(INFO) << "Initializing ChunkServer modules";
 
+    LOG_IF(FATAL, !conf.GetUInt32Value("global.min_io_alignment",
+                                       &FLAGS_minIoAlignment))
+        << "Failed to get global.min_io_alignment";
+    LOG_IF(FATAL, !common::is_aligned(FLAGS_minIoAlignment, 512))
+        << "minIoAlignment should align to 512";
+
     // 优先初始化 metric 收集模块
     ChunkServerMetricOptions metricOptions;
     InitMetricOptions(&conf, &metricOptions);
@@ -559,6 +565,8 @@ void ChunkServer::InitCopysetNodeOptions(
         &copysetNodeOptions->maxChunkSize));
     LOG_IF(FATAL, !conf->GetUInt32Value("global.location_limit",
         &copysetNodeOptions->locationLimit));
+    LOG_IF(FATAL, !conf->GetUInt32Value("global.meta_page_size",
+        &copysetNodeOptions->pageSize));
     LOG_IF(FATAL, !conf->GetUInt32Value("copyset.load_concurrency",
         &copysetNodeOptions->loadConcurrency));
     LOG_IF(FATAL, !conf->GetUInt32Value("copyset.check_retrytimes",
@@ -821,6 +829,10 @@ void ChunkServer::LoadConfigFromCmdline(common::Configuration *conf) {
         conf->SetIntValue("copyset.load_concurrency",
             FLAGS_copysetLoadConcurrency);
     }
+
+    if (GetCommandLineFlagInfo("minIoAlignment", &info) && !info.is_default) {
+        conf->SetUInt32Value("global.min_io_alignment", FLAGS_minIoAlignment);
+    }
 }
 
 int ChunkServer::GetChunkServerMetaFromLocal(