[UPDATED MY ANSWER]
etc/adminhtml/di.xml
Add modifier to add custom form modal
<virtualType name="Vendor\Module\Ui\DataProvider\Item\Form\Modifier\Pool" type="Magento\Ui\DataProvider\Modifier\Pool">
<arguments>
<argument name="modifiers" xsi:type="array">
<item name="item-form-modal" xsi:type="array">
<item name="class" xsi:type="string">Vendor\Module\Ui\DataProvider\Item\Form\Modifier\Item</item>
<item name="sortOrder" xsi:type="number">10</item>
</item>
</argument>
</arguments>
</virtualType>
Button action for render modal
return [
'label' => __('Add New Item'),
'class' => 'primary button-add-item',
'data_attribute' => [
'mage-init' => [
'Magento_Ui/js/form/button-adapter' => [
'actions' => [
[
'targetName' => 'item_form.item_form.item_form_modal',
'actionName' => 'toggleModal'
],
[
'targetName' => 'item_form.item_form.item_form_modal.insert_item_form',
'actionName' => 'render'
]
]
]
]
],
'on_click' => '',
'sort_order' => 20
];
}
File Modifier where i defination form
return $this->arrayManager->set(
'item_form_modal',
$meta,
[
'arguments' => [
'data' => [
'config' => [
'isTemplate' => false,
'componentType' => 'modal',
'options' => [
'title' => __('New Item'),
],
'imports' => [
'state' => '!index=insert_item_form:responseStatus'
],
],
],
],
'children' => [
'insert_item_form' => [
'arguments' => [
'data' => [
'config' => [
'label' => '',
'componentType' => 'container',
'component' => 'Magento_Ui/js/form/components/insert-form',
'dataScope' => '',
'update_url' => $this->urlBuilder->getUrl('mui/index/render'),
'render_url' => $this->urlBuilder->getUrl(
'mui/index/render_handle',
[
'handle' => 'megamenuadmin_item_form',
'store' => $this->_storeManager->getStore()->getId(),
'buttons' => 1
]
),
'autoRender' => false,
'ns' => 'item_add_form',
'externalProvider' => 'item_add_form.item_add_form_data_source',
'toolbarContainer' => '${ $.parentName }',
'formSubmitType' => 'ajax',
],
],
]
]
]
]
);
item_form_modal
has child element form ui insert_item_form
insert via ajax. Handle layout megamenuadmin_item_form
contain ui-form
. Between children [] you can add any components such as: fieldsets, fields, selects, inputs v.v or ui-form like i do
layout megamenuadmin_item_form
<page>
<referenceContainer name="content">
<uiComponent name="item_add_form"/>
</referenceContainer>
</page>
form ui-component item_add_form
<?xml version="1.0" encoding="UTF-8"?>
<form xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:module:Magento_Ui:etc/ui_configuration.xsd">
<argument name="data" xsi:type="array">
<item name="js_config" xsi:type="array">
<item name="provider" xsi:type="string">item_add_form.item_add_form_data_source</item>
<item name="deps" xsi:type="string">item_add_form.item_add_form_data_source</item>
<item name="namespace" xsi:type="string">item_add_form</item>
</item>
<item name="buttons" xsi:type="array">
<item name="cancel" xsi:type="string">Vendor\Module\Block\Adminhtml\Item\Button\Cancel</item>
<item name="save" xsi:type="string">Vendor\Module\Block\Adminhtml\Item\Button\Save</item>
</item>
<item name="template" xsi:type="string">templates/form/collapsible</item>
<item name="config" xsi:type="array">
<item name="ajaxSaveType" xsi:type="string">simple</item>
</item>
</argument>
<dataSource name="item_add_form_data_source">
<argument name="dataProvider" xsi:type="configurableObject">
<argument name="class" xsi:type="string">Vendor\Module\Model\Item\DataProvider</argument>
<argument name="name" xsi:type="string">item_add_form_data_source</argument>
<argument name="primaryFieldName" xsi:type="string">id</argument>
<argument name="requestFieldName" xsi:type="string">menu_id</argument>
<argument name="data" xsi:type="array">
<item name="config" xsi:type="array">
<item name="submit_url" xsi:type="url" path="megamenuadmin/item/save"/>
</item>
</argument>
</argument>
<argument name="data" xsi:type="array">
<item name="js_config" xsi:type="array">
<item name="component" xsi:type="string">Magento_Ui/js/form/provider</item>
</item>
</argument>
</dataSource>
<fieldset name="filedset1">
Fields go here
</fieldset>
<fieldset name="fieldset2">
</fieldset>
</form>