feat: 【数据源】API数据源支持分页获取数据

2026-05-14 21:12:33 +08:00 · 2024-12-18 16:09:10 +08:00
parent 645bc0ddec
commit ea5f0f4156
6 changed files with 178 additions and 40 deletions
--- a/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java
+++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/ApiUtils.java
@@ -12,10 +12,7 @@ import io.dataease.exception.DEException;
 import io.dataease.extensions.datasource.dto.DatasetTableDTO;
 import io.dataease.extensions.datasource.dto.DatasourceRequest;
 import io.dataease.extensions.datasource.dto.TableField;
-import io.dataease.utils.CommonBeanFactory;
-import io.dataease.utils.HttpClientConfig;
-import io.dataease.utils.HttpClientUtil;
-import io.dataease.utils.JsonUtil;
+import io.dataease.utils.*;
 import org.apache.commons.lang3.StringUtils;
 import org.json.simple.JSONArray;
 import org.springframework.util.CollectionUtils;
@@ -82,12 +79,49 @@ public class ApiUtils {
        if (apiDefinition == null) {
            DEException.throwException("未找到");
        }
-        String response = execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
-        fieldList = getTableFields(apiDefinition);
-        result.put("fieldList", fieldList);
-        dataList = fetchResult(response, apiDefinition);
-        result.put("dataList", dataList);
-        return result;
+        if (apiDefinition.getRequest().getPage() != null && !apiDefinition.getRequest().getPage().getPageType().equalsIgnoreCase("empty")) {
+            String response = execHttpRequest(false, apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
+            fieldList = getTableFields(apiDefinition);
+            result.put("fieldList", fieldList);
+            if (apiDefinition.getRequest().getPage().getPageType().equalsIgnoreCase("pageNumber")) {
+                int pageCount = JsonPath.read(response, apiDefinition.getRequest().getPage().getResponseData().get(0).getResolutionPath());
+                if (apiDefinition.getRequest().getPage().getResponseData().get(0).getResolutionPathType().equalsIgnoreCase("totalNumber")) {
+                    pageCount = pageCount / Integer.valueOf(apiDefinition.getRequest().getPage().getRequestData().get(1).getParameterDefaultValue());
+                }
+                for (int i = 1; i < pageCount + 1; i++) {
+                    apiDefinition.getRequest().getPage().getRequestData().get(0).setParameterDefaultValue(String.valueOf(i));
+                    response = execHttpRequest(false, apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
+                    dataList.addAll(fetchResult(response, apiDefinition));
+                }
+            }
+            if (apiDefinition.getRequest().getPage().getPageType().equalsIgnoreCase("cursor")) {
+                dataList.addAll(fetchResult(response, apiDefinition));
+                String cursor = null;
+                try {
+                    cursor = JsonPath.read(response, apiDefinition.getRequest().getPage().getResponseData().get(0).getResolutionPath());
+                } catch (Exception e) {
+                }
+                while (cursor != null) {
+                    apiDefinition.getRequest().getPage().getRequestData().get(0).setParameterDefaultValue(cursor);
+                    response = execHttpRequest(false, apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
+                    dataList.addAll(fetchResult(response, apiDefinition));
+                    try {
+                        cursor = JsonPath.read(response, apiDefinition.getRequest().getPage().getResponseData().get(0).getResolutionPath());
+                    } catch (Exception e) {
+                        cursor = null;
+                    }
+                }
+            }
+            result.put("dataList", dataList);
+            return result;
+        } else {
+            String response = execHttpRequest(false, apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
+            fieldList = getTableFields(apiDefinition);
+            result.put("fieldList", fieldList);
+            dataList = fetchResult(response, apiDefinition);
+            result.put("dataList", dataList);
+            return result;
+        }
    }


@@ -142,11 +176,26 @@ public class ApiUtils {
        if (apiDefinition == null) {
            DEException.throwException("未找到");
        }
-        String response = execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
+        String response = execHttpRequest(true, apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), params(datasourceRequest));
        return fetchResult(response, apiDefinition);
    }

-    public static String execHttpRequest(ApiDefinition apiDefinition, int socketTimeout, List<ApiDefinition> paramsList) {
+    public static String execHttpRequest(boolean preview, ApiDefinition api, int socketTimeout, List<ApiDefinition> paramsList) {
+        ApiDefinition apiDefinition = new ApiDefinition();
+        BeanUtils.copyBean(apiDefinition, api);
+
+        if (apiDefinition.getRequest().getPage() != null && apiDefinition.getRequest().getPage().getPageType() != null && apiDefinition.getRequest().getPage().getPageType().equalsIgnoreCase("pageNumber")) {
+            apiDefinition.setUrl(apiDefinition.getUrl().replace(apiDefinition.getRequest().getPage().getRequestData().get(0).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(0).getParameterDefaultValue()).replace(apiDefinition.getRequest().getPage().getRequestData().get(1).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(1).getParameterDefaultValue()));
+            apiDefinition.setRequest(JsonUtil.parseObject(JsonUtil.toJSONString(apiDefinition.getRequest()).toString().replace(apiDefinition.getRequest().getPage().getRequestData().get(0).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(0).getParameterDefaultValue()).replace(apiDefinition.getRequest().getPage().getRequestData().get(1).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(1).getParameterDefaultValue()), ApiDefinitionRequest.class));
+        }
+
+        if (apiDefinition.getRequest().getPage() != null && apiDefinition.getRequest().getPage().getPageType() != null && apiDefinition.getRequest().getPage().getPageType().equalsIgnoreCase("cursor")) {
+            apiDefinition.setUrl(apiDefinition.getUrl().replace(apiDefinition.getRequest().getPage().getRequestData().get(0).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(0).getParameterDefaultValue()).replace(apiDefinition.getRequest().getPage().getRequestData().get(1).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(1).getParameterDefaultValue()));
+            String defaultCursor = apiDefinition.getRequest().getPage().getRequestData().get(0).getParameterDefaultValue();
+            apiDefinition.setRequest(JsonUtil.parseObject(JsonUtil.toJSONString(apiDefinition.getRequest()).toString().replace(apiDefinition.getRequest().getPage().getRequestData().get(0).getBuiltInParameterName(), StringUtils.isEmpty(defaultCursor) ? "" : defaultCursor).replace(apiDefinition.getRequest().getPage().getRequestData().get(1).getBuiltInParameterName(), apiDefinition.getRequest().getPage().getRequestData().get(1).getParameterDefaultValue()), ApiDefinitionRequest.class));
+        }
+
+
        String response = "";
        HttpClientConfig httpClientConfig = new HttpClientConfig();
        httpClientConfig.setSocketTimeout(socketTimeout * 1000);
@@ -159,7 +208,7 @@ public class ApiUtils {
                        for (int i = 0; i < definition.getFields().size(); i++) {
                            TableField field = definition.getFields().get(i);
                            if (field.getOriginName().equalsIgnoreCase(param)) {
-                                String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
+                                String resultStr = execHttpRequest(true, definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
                                List<String[]> dataList = fetchResult(resultStr, definition);
                                if (dataList.size() > 0) {
                                    httpClientConfig.addHeader(header.get("name").toString(), dataList.get(0)[i]);
@@ -181,7 +230,7 @@ public class ApiUtils {
                            for (int i = 0; i < definition.getFields().size(); i++) {
                                TableField field = definition.getFields().get(i);
                                if (field.getOriginName().equalsIgnoreCase(param)) {
-                                    String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
+                                    String resultStr = execHttpRequest(true, definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
                                    List<String[]> dataList = fetchResult(resultStr, definition);
                                    if (dataList.size() > 0) {
                                        result = result.replace("${" + param + "}", dataList.get(0)[i]);
@@ -223,7 +272,7 @@ public class ApiUtils {
                        for (int i = 0; i < definition.getFields().size(); i++) {
                            TableField field = definition.getFields().get(i);
                            if (field.getOriginName().equalsIgnoreCase(param)) {
-                                String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
+                                String resultStr = execHttpRequest(true, definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
                                List<String[]> dataList = fetchResult(resultStr, definition);
                                if (dataList.size() > 0) {
                                    params.add(argument.get("name") + "=" + dataList.get(0)[i]);
@@ -245,7 +294,7 @@ public class ApiUtils {
                            for (int i = 0; i < definition.getFields().size(); i++) {
                                TableField field = definition.getFields().get(i);
                                if (field.getOriginName().equalsIgnoreCase(param)) {
-                                    String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
+                                    String resultStr = execHttpRequest(true, definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
                                    List<String[]> dataList = fetchResult(resultStr, definition);
                                    if (dataList.size() > 0) {
                                        result = result.replace("${" + param + "}", dataList.get(0)[i]);
@@ -274,7 +323,6 @@ public class ApiUtils {

        switch (apiDefinition.getMethod()) {
            case "GET":
-
                response = HttpClientUtil.get(apiDefinition.getUrl().trim(), httpClientConfig);
                break;
            case "POST":
@@ -310,7 +358,7 @@ public class ApiUtils {
                                            for (int i = 0; i < definition.getFields().size(); i++) {
                                                TableField field = definition.getFields().get(i);
                                                if (field.getOriginName().equalsIgnoreCase(param)) {
-                                                    String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
+                                                    String resultStr = execHttpRequest(false, definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
                                                    List<String[]> dataList = fetchResult(resultStr, definition);
                                                    if (dataList.size() > 0) {
                                                        body.put(jsonNode.get("name").toString(), dataList.get(0)[i]);
@@ -332,7 +380,7 @@ public class ApiUtils {
                                                for (int i = 0; i < definition.getFields().size(); i++) {
                                                    TableField field = definition.getFields().get(i);
                                                    if (field.getOriginName().equalsIgnoreCase(param)) {
-                                                        String resultStr = execHttpRequest(definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
+                                                        String resultStr = execHttpRequest(false, definition, definition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), null);
                                                        List<String[]> dataList = fetchResult(resultStr, definition);
                                                        if (dataList.size() > 0) {
                                                            result = result.replace("${" + param + "}", dataList.get(0)[i]);
--- a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
+++ b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
@@ -876,10 +876,9 @@ public class DatasourceServer implements DatasourceApi {
    }

    public ApiDefinition checkApiDatasource(Map<String, String> request) throws DEException {
-
        ApiDefinition apiDefinition = JsonUtil.parseObject(new String(java.util.Base64.getDecoder().decode(request.get("data"))), ApiDefinition.class);
        List<ApiDefinition> paramsList = JsonUtil.parseList(new String(java.util.Base64.getDecoder().decode(request.get("paramsList"))), listTypeReference);
-        String response = ApiUtils.execHttpRequest(apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), paramsList);
+        String response = ApiUtils.execHttpRequest(true, apiDefinition, apiDefinition.getApiQueryTimeout() == null || apiDefinition.getApiQueryTimeout() <= 0 ? 10 : apiDefinition.getApiQueryTimeout(), paramsList);
        if (request.keySet().contains("type") && request.get("type").equals("apiStructure")) {
            apiDefinition.setShowApiStructure(true);
        }
--- a/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestForm.vue
+++ b/core/core-frontend/src/views/visualized/data/datasource/form/ApiHttpRequestForm.vue
@@ -10,6 +10,7 @@ import { Body } from './ApiTestModel.js'
 import type { Item } from './ApiKeyValue.vue'
 import type { AuthConfig } from './ApiAuthConfig.vue'
 import type { ApiBodyItem } from './ApiBody.vue'
+import { PageSetting } from '@/views/visualized/data/datasource/form/Pagination.vue'
 export interface ApiRequest {
  changeId: string
  headers: Item[]
@@ -17,6 +18,7 @@ export interface ApiRequest {
  arguments: Item[]
  authManager: AuthConfig
  body: ApiBodyItem
+  page: PageSetting
 }
 const props = defineProps({
  showScript: {
@@ -38,6 +40,11 @@ const props = defineProps({
      body: {
        typeChange: '',
        kvs: []
+      },
+      page: {
+        pageType: 'empty',
+        requestData: [],
+        responseData: []
      }
    })
  },
@@ -193,7 +200,7 @@ const emits = defineEmits(['changeId'])
        <api-auth-config :request="apiRequest" />
      </el-tab-pane>
      <el-tab-pane key="pagination" :label="t('api_pagination.paging_ettings')" name="pagination">
-        <Pagination />
+        <Pagination :page="apiRequest.page" />
      </el-tab-pane>
    </el-tabs>
  </div>
--- a/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue
+++ b/core/core-frontend/src/views/visualized/data/datasource/form/EditorDetail.vue
@@ -110,6 +110,11 @@ const defaultApiItem = {
      raw: '',
      kvs: []
    },
+    page: {
+      pageType: 'empty',
+      requestData: [],
+      responseData: []
+    },
    authManager: {
      verification: '',
      username: '',
--- a/core/core-frontend/src/views/visualized/data/datasource/form/Pagination.vue
+++ b/core/core-frontend/src/views/visualized/data/datasource/form/Pagination.vue
@@ -1,17 +1,45 @@
 <script lang="ts" setup>
-import { ref } from 'vue'
+import { onBeforeMount, PropType, ref, toRefs } from 'vue'
 import { cloneDeep } from 'lodash-es'
 import { useI18n } from '@/hooks/web/useI18n'

+export interface PageSetting {
+  pageType: string
+  requestData: requestItem[]
+  responseData: responseItem[]
+}
+
+export interface requestItem {
+  parameterName: string
+  builtInParameterName: string
+  requestParameterName: string
+  parameterDefaultValue: string
+}
+
+export interface responseItem {
+  parameterName: string
+  resolutionPath: string
+  resolutionPathType: string
+}
+const props = defineProps({
+  page: {
+    type: Object as PropType<PageSetting>,
+    default: () => ({
+      pageType: '',
+      requestData: [],
+      responseData: []
+    })
+  }
+})
+const { page } = toRefs(props)
 const { t } = useI18n()
-const numberToken = ref('empty')
 const options = [
  {
-    value: 'number',
+    value: 'pageNumber',
    label: t('api_pagination.number__size')
  },
  {
-    value: 'token',
+    value: 'cursor',
    label: t('api_pagination.cursor__size')
  },
  {
@@ -37,15 +65,22 @@ const requestData = ref([

 const defaultPathArr = [
  {
-    value: 'number',
+    value: 'totalNumber',
    label: t('api_pagination.total_number_de')
  },
  {
-    value: 'page',
+    value: 'totalPage',
    label: t('api_pagination.number_of_pages')
  }
 ]

+const cursorPathArr = [
+  {
+    value: 'cursor',
+    label: t('api_pagination.cursor')
+  }
+]
+
 const resolutionPathOptions = ref(cloneDeep(defaultPathArr))

 const responseData = ref([
@@ -56,21 +91,40 @@ const responseData = ref([
  }
 ])

+onBeforeMount(() => {
+  if (page.value.requestData.length === 0) {
+    page.value.requestData = requestData.value
+  }
+  if (page.value.responseData.length === 0) {
+    page.value.responseData = responseData.value
+  }
+  if (page.value.pageType === '') {
+    // page.value.pageType = 'empty'
+  }
+})
+
 const handleNumberSizeChange = () => {
-  resolutionPathOptions.value =
-    numberToken.value === 'number' ? cloneDeep(defaultPathArr) : [cloneDeep(defaultPathArr)[0]]
-  responseData.value[0].resolutionPathType = 'number'
-  requestData.value[0].parameterName =
-    numberToken.value === 'number' ? t('api_pagination.page_number') : t('api_pagination.cursor')
-  responseData.value[0].parameterName =
-    numberToken.value === 'number' ? t('api_pagination.total_number') : t('api_pagination.cursor')
+  if (page.value.pageType === 'pageNumber') {
+    page.value.responseData[0].resolutionPathType = 'totalNumber'
+    page.value.responseData[0].parameterName = t('api_pagination.total_number')
+    resolutionPathOptions.value = cloneDeep(defaultPathArr)
+    page.value.requestData[0].parameterName = t('api_pagination.page_number')
+    page.value.requestData[0].builtInParameterName = '${pageNumber}'
+  }
+  if (page.value.pageType === 'cursor') {
+    page.value.responseData[0].resolutionPathType = 'cursor'
+    page.value.responseData[0].parameterName = t('api_pagination.cursor')
+    resolutionPathOptions.value = cloneDeep(cursorPathArr)
+    page.value.requestData[0].parameterName = t('api_pagination.cursor')
+    page.value.requestData[0].builtInParameterName = '${pageToken}'
+  }
 }
 </script>

 <template>
  <div class="api-pagination">
    <span class="type">{{ t('api_pagination.pagination_method') }}</span>
-    <el-select v-model="numberToken" @change="handleNumberSizeChange" style="width: 100%">
+    <el-select v-model="page.pageType" @change="handleNumberSizeChange" style="width: 100%">
      <el-option
        v-for="item in options"
        :key="item.value"
@@ -78,9 +132,9 @@ const handleNumberSizeChange = () => {
        :value="item.value"
      />
    </el-select>
-    <template v-if="numberToken !== 'empty'">
+    <template v-if="page.pageType !== 'empty'">
      <div class="table-title request">{{ t('datasource.request') }}</div>
-      <el-table header-cell-class-name="header-cell" :data="requestData" style="width: 100%">
+      <el-table header-cell-class-name="header-cell" :data="page.requestData" style="width: 100%">
        <el-table-column prop="parameterName" :label="t('api_pagination.parameter_name')" />
        <el-table-column
          prop="builtInParameterName"
@@ -105,8 +159,9 @@ const handleNumberSizeChange = () => {
          </template>
        </el-table-column>
      </el-table>
+
      <div class="table-title response">{{ t('api_pagination.response') }}</div>
-      <el-table header-cell-class-name="header-cell" :data="responseData" style="width: 100%">
+      <el-table header-cell-class-name="header-cell" :data="page.responseData" style="width: 100%">
        <el-table-column
          prop="parameterName"
          :label="t('api_pagination.parameter_name')"
--- a/sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ApiDefinitionRequest.java
+++ b/sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ApiDefinitionRequest.java
@@ -15,12 +15,36 @@ public class ApiDefinitionRequest {
    private List<Map<String, String>> arguments = new ArrayList<>();
    private List<Map<String, String>> rest = new ArrayList<>();
    private AuthManager authManager = new AuthManager();
+    private Page page = new Page();


    @Data
-    public static class AuthManager{
+    public static class AuthManager {
        private String password;
        private String username;
        private String verification = "";
    }
+
+    @Data
+    public static class Page {
+        private String pageType;
+        private List<RequestItem> requestData;
+        private List<ResponseItem> responseData;
+    }
+
+    @Data
+    public static class RequestItem {
+        private String parameterName;
+        private String builtInParameterName;
+        private String requestParameterName;
+        private String parameterDefaultValue;
+    }
+
+    @Data
+    public static class ResponseItem {
+        private String parameterName;
+        private String resolutionPath;
+        private String resolutionPathType;
+    }
+
 }