Final Edit:

Solution based on passing overlayLib as an attribute to dynamically created modal component.

Last idea that I had (and tested) is to pass overlayLib as an attribute when dynamically creating modal:

    <aura:component implements="flexipage:availableForAllPageTypes">
        <aura:attribute name="lib" type="Object"/>
        <lightning:fileUpload  name="fileUploader"
                               multiple="false"
                               accept="{!v.fileExtensions}"
                               recordId="{!v.relatedRecordId}"
                               onuploadfinished="{!c.handleUploadFinished }"/>  
    </aura:component>


    handleUploadFinished: function (component, event) {
        var uploadedFiles = event.getParam("files");
        console.log(uploadedFiles[0].documentId); // See? it works.
        component.get("v.lib").notifyClose();
    }

Creating modal

    handleShowModalFooter : function (component, event, helper) {
        var modalBody;
        var modalFooter;
        $A.createComponents([
            ["c:modalContent",{'lib':  component.find('overlayLib')}],
            ["c:modalFooter",{}]
        ],
        function(components, status){
            if (status === "SUCCESS") {
                modalBody = components[0];
                modalFooter = components[1];
                component.find('overlayLib').showCustomModal({
                   header: "Application Confirmation",
                   body: modalBody, 
                   footer: modalFooter,
                   showCloseButton: true,
                   cssClass: "my-modal,my-custom-class,my-other-class",
                   closeCallback: function() {
                       alert('You closed the alert!');
                   }
               })
            }
        }
       );
    }





Edit: Ok I didnt fully understand question, so I'm adding new answer with old one staying just for reference.

You can of course create footer for your modal with buttons that will handle modal close.

Example mainly from the docs:

Modal footer component: 

    <aura:component>
        <lightning:overlayLibrary aura:id="overlayLib"/>
        <lightning:button name="cancel" label="Cancel" onclick="{!c.handleCancel}"/>
        <lightning:button name="Close" label="Close" variant="brand" onclick="{!c.handleCancel}" disabled="true"/>
    </aura:component>

Footer controller:

    ({
        handleCancel : function(component, event, helper) {
            //closes the modal or popover from the component
            component.find("overlayLib").notifyClose();
        }
    })

Creating modal controller:

    ({
    handleShowModalFooter : function (component, event, helper) {
        var modalBody;
        var modalFooter;
        $A.createComponents([
            ["c:modalContent",{}],
            ["c:modalFooter",{}]
        ],
        function(components, status){
            if (status === "SUCCESS") {
                modalBody = components[0];
                modalFooter = components[1];
                component.find('overlayLib').showCustomModal({
                   header: "Application Confirmation",
                   body: modalBody, 
                   footer: modalFooter,
                   showCloseButton: true,
                   cssClass: "my-modal,my-custom-class,my-other-class",
                   closeCallback: function() {
                       alert('You closed the alert!');
                   }
               })
            }
        }
       );
    }
    })

I on purpose made one button disabled in a footer so you can enable that in your code