+ {{^_disableToc}} + {{>partials/toc}} +
+ {{/_disableToc}} + {{#_disableToc}} +
+ {{/_disableToc}} + {{#_disableAffix}} +
+ {{/_disableAffix}} + {{^_disableAffix}} +
+ {{/_disableAffix}} +
+ {{!body}} +
+
+ {{^_disableAffix}} + {{>partials/affix}} + {{/_disableAffix}} +
+
+ {{^_disableFooter}} + {{>partials/footer}} + {{/_disableFooter}} +
+ {{>partials/scripts}} + + diff --git a/csharp/ApiDocs/_exported_templates/default/logo.svg b/csharp/ApiDocs/_exported_templates/default/logo.svg new file mode 100644 index 0000000000..51519427aa --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/logo.svg @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_affix.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_affix.liquid new file mode 100644 index 0000000000..d06b0361cf --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_affix.liquid @@ -0,0 +1,25 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_breadcrumb.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_breadcrumb.liquid new file mode 100644 index 0000000000..3bf0d72af9 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_breadcrumb.liquid @@ -0,0 +1,8 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} +
+ +
diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_footer.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_footer.liquid new file mode 100644 index 0000000000..5d5377748e --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_footer.liquid @@ -0,0 +1,11 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_head.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_head.liquid new file mode 100644 index 0000000000..2af71a645e --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_head.liquid @@ -0,0 +1,38 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} + + + + {%- if title and _appTitle -%} + {{title}} | {{appTitle}} + + {%- else -%} + {%- if title or _appTitle -%} + {{title}}{{appTitle}} + + {%- endif -%} + {%- endif -%} + + + {%- if _description -%} + + {%- endif -%} + {%- if _appFaviconPath -%} + + {%- else -%} + + {%- endif -%} + + + + + + {%- if _noindex -%} + + {%- endif -%} + {%- if _enableSearch -%} + + {%- endif -%} + {%- if _enableNewTab -%} + + {%- endif -%} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_logo.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_logo.liquid new file mode 100644 index 0000000000..7bc65a65b8 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_logo.liquid @@ -0,0 +1,8 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} + + {%- if _appLogoPath -%} + + {%- else -%} + + {%- endif -%} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_navbar.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_navbar.liquid new file mode 100644 index 0000000000..bcfde283c7 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_navbar.liquid @@ -0,0 +1,21 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_scripts.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_scripts.liquid new file mode 100644 index 0000000000..6640a85cdf --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_scripts.liquid @@ -0,0 +1,4 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} + + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/_toc.liquid b/csharp/ApiDocs/_exported_templates/default/partials/_toc.liquid new file mode 100644 index 0000000000..ea6706ee27 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/_toc.liquid @@ -0,0 +1,7 @@ +{% comment -%}Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.{% endcomment -%} +
+ +
+
+
+
diff --git a/csharp/ApiDocs/_exported_templates/default/partials/affix.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/affix.tmpl.partial new file mode 100644 index 0000000000..c2882e3845 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/affix.tmpl.partial @@ -0,0 +1,26 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/breadcrumb.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/breadcrumb.tmpl.partial new file mode 100644 index 0000000000..f63f7e3368 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/breadcrumb.tmpl.partial @@ -0,0 +1,9 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +
+ +
diff --git a/csharp/ApiDocs/_exported_templates/default/partials/class.header.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/class.header.tmpl.partial new file mode 100644 index 0000000000..1a732fb9a2 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/class.header.tmpl.partial @@ -0,0 +1,121 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +

{{>partials/title}}

+
{{{summary}}}
+
{{{conceptual}}}
+{{#inClass}} +
+
{{__global.inheritance}}
+ {{#inheritance}} +
{{{specName.0.value}}}
+ {{/inheritance}} +
{{name.0.value}}
+ {{#derivedClasses}} +
{{{specName.0.value}}}
+ {{/derivedClasses}} +
+{{/inClass}} +{{#implements.0}} +
+
{{__global.implements}}
+{{/implements.0}} +{{#implements}} +
{{{specName.0.value}}}
+{{/implements}} +{{#implements.0}} +
+{{/implements.0}} +{{#inheritedMembers.0}} +
+
{{__global.inheritedMembers}}
+{{/inheritedMembers.0}} +{{#inheritedMembers}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/inheritedMembers}} +{{#inheritedMembers.0}} +
+{{/inheritedMembers.0}} +
{{__global.namespace}}: {{{namespace.specName.0.value}}}
+
{{__global.assembly}}: {{assemblies.0}}.dll
+
{{__global.syntax}}
+
+ 
{{syntax.content.0.value}}
+
+{{#syntax.parameters.0}} +
{{__global.parameters}}
+ + + + + + + + + +{{/syntax.parameters.0}} +{{#syntax.parameters}} + + + + + +{{/syntax.parameters}} +{{#syntax.parameters.0}} + +
{{__global.type}}{{__global.name}}{{__global.description}}
{{{type.specName.0.value}}}{{{id}}}{{{description}}}
+{{/syntax.parameters.0}} +{{#syntax.return}} +
{{__global.returns}}
+ + + + + + + + + + + + + +
{{__global.type}}{{__global.description}}
{{{type.specName.0.value}}}{{{description}}}
+{{/syntax.return}} +{{#syntax.typeParameters.0}} +
{{__global.typeParameters}}
+ + + + + + + + +{{/syntax.typeParameters.0}} +{{#syntax.typeParameters}} + + + + +{{/syntax.typeParameters}} +{{#syntax.typeParameters.0}} + +
{{__global.name}}{{__global.description}}
{{{id}}}{{{description}}}
+{{/syntax.typeParameters.0}} +{{#remarks}} +
{{__global.remarks}}
+
{{{remarks}}}
+{{/remarks}} +{{#example.0}} +
{{__global.examples}}
+{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/class.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/class.tmpl.partial new file mode 100644 index 0000000000..7490fbe40a --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/class.tmpl.partial @@ -0,0 +1,234 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +{{>partials/class.header}} +{{#children}} +

{{>partials/classSubtitle}}

+{{#children}} +{{^_disableContribution}} +{{#docurl}} + + | + {{__global.improveThisDoc}} +{{/docurl}} +{{#sourceurl}} + + {{__global.viewSource}} +{{/sourceurl}} +{{/_disableContribution}} +{{#overload}} + +{{/overload}} +

{{name.0.value}}

+
{{{summary}}}
+
{{{conceptual}}}
+
{{__global.declaration}}
+{{#syntax}} +
+ 
{{syntax.content.0.value}}
+
+{{#parameters.0}} +
{{__global.parameters}}
+ + + + + + + + + +{{/parameters.0}} +{{#parameters}} + + + + + +{{/parameters}} +{{#parameters.0}} + +
{{__global.type}}{{__global.name}}{{__global.description}}
{{{type.specName.0.value}}}{{{id}}}{{{description}}}
+{{/parameters.0}} +{{#return}} +
{{__global.returns}}
+ + + + + + + + + + + + + +
{{__global.type}}{{__global.description}}
{{{type.specName.0.value}}}{{{description}}}
+{{/return}} +{{#typeParameters.0}} +
{{__global.typeParameters}}
+ + + + + + + + +{{/typeParameters.0}} +{{#typeParameters}} + + + + +{{/typeParameters}} +{{#typeParameters.0}} + +
{{__global.name}}{{__global.description}}
{{{id}}}{{{description}}}
+{{/typeParameters.0}} +{{#fieldValue}} +
{{__global.fieldValue}}
+ + + + + + + + + + + + + +
{{__global.type}}{{__global.description}}
{{{type.specName.0.value}}}{{{description}}}
+{{/fieldValue}} +{{#propertyValue}} +
{{__global.propertyValue}}
+ + + + + + + + + + + + + +
{{__global.type}}{{__global.description}}
{{{type.specName.0.value}}}{{{description}}}
+{{/propertyValue}} +{{#eventType}} +
{{__global.eventType}}
+ + + + + + + + + + + + + +
{{__global.type}}{{__global.description}}
{{{type.specName.0.value}}}{{{description}}}
+{{/eventType}} +{{/syntax}} +{{#overridden}} +
{{__global.overrides}}
+
+{{/overridden}} +{{#remarks}} +
{{__global.remarks}}
+
{{{remarks}}}
+{{/remarks}} +{{#example.0}} +
{{__global.examples}}
+{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} +{{#exceptions.0}} +
{{__global.exceptions}}
+ + + + + + + + +{{/exceptions.0}} +{{#exceptions}} + + + + +{{/exceptions}} +{{#exceptions.0}} + +
{{__global.type}}{{__global.condition}}
{{{type.specName.0.value}}}{{{description}}}
+{{/exceptions.0}} +{{#seealso.0}} +
{{__global.seealso}}
+
+{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
{{{type.specName.0.value}}}
+ {{/isCref}} + {{^isCref}} +
{{{url}}}
+ {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
+{{/seealso.0}} +{{/children}} +{{/children}} +{{#implements.0}} +

{{__global.implements}}

+{{/implements.0}} +{{#implements}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/implements}} +{{#extensionMethods.0}} +

{{__global.extensionMethods}}

+{{/extensionMethods.0}} +{{#extensionMethods}} +
+ {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
+{{/extensionMethods}} +{{#seealso.0}} +

{{__global.seealso}}

+
+{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
{{{type.specName.0.value}}}
+ {{/isCref}} + {{^isCref}} +
{{{url}}}
+ {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
+{{/seealso.0}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/classSubtitle.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/classSubtitle.tmpl.partial new file mode 100644 index 0000000000..5caea3656b --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/classSubtitle.tmpl.partial @@ -0,0 +1,28 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} +{{#inConstructor}} +{{__global.constructorsInSubtitle}} +{{/inConstructor}} +{{#inField}} +{{__global.fieldsInSubtitle}} +{{/inField}} +{{#inProperty}} +{{__global.propertiesInSubtitle}} +{{/inProperty}} +{{#inMethod}} +{{__global.methodsInSubtitle}} +{{/inMethod}} +{{#inEvent}} +{{__global.eventsInSubtitle}} +{{/inEvent}} +{{#inOperator}} +{{__global.operatorsInSubtitle}} +{{/inOperator}} +{{#inEii}} +{{__global.eiisInSubtitle}} +{{/inEii}} +{{#inFunction}} +{{__global.functionsInSubtitle}} +{{/inFunction}} +{{#inMember}} +{{__global.membersInSubtitle}} +{{/inMember}} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/partials/customMREFContent.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/customMREFContent.tmpl.partial new file mode 100644 index 0000000000..67395b8930 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/customMREFContent.tmpl.partial @@ -0,0 +1,2 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} +{{!Add your own custom template for the content for ManagedReference here}} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/partials/dd-li.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/dd-li.tmpl.partial new file mode 100644 index 0000000000..75636c29b5 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/dd-li.tmpl.partial @@ -0,0 +1,3 @@ +{{#items}} +
  • {{name}}
    • +{{/items}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/enum.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/enum.tmpl.partial new file mode 100644 index 0000000000..b5584adfaf --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/enum.tmpl.partial @@ -0,0 +1,50 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +{{>partials/class.header}} +{{#children}} +

    {{>partials/classSubtitle}}

    + + + + + + + + + {{#children}} + + + + + {{/children}} + +
    {{__global.name}}{{__global.description}}
    {{name.0.value}}{{{summary}}}
    +{{/children}} +{{#seealso.0}} +
    {{__global.seealso}}
    +
    +{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
    {{{type.specName.0.value}}}
    + {{/isCref}} + {{^isCref}} +
    {{{url}}}
    + {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
    +{{/seealso.0}} +{{#extensionMethods.0}} +

    {{__global.extensionMethods}}

    +{{/extensionMethods.0}} +{{#extensionMethods}} +
    + {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
    +{{/extensionMethods}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/footer.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/footer.tmpl.partial new file mode 100644 index 0000000000..06bad87e85 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/footer.tmpl.partial @@ -0,0 +1,12 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/head.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/head.tmpl.partial new file mode 100644 index 0000000000..4e8cd7918b --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/head.tmpl.partial @@ -0,0 +1,20 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + + + + + {{#title}}{{title}}{{/title}}{{^title}}{{>partials/title}}{{/title}} {{#_appTitle}}| {{_appTitle}} {{/_appTitle}} + + + + {{#_description}}{{/_description}} + + + + + + + {{#_noindex}}{{/_noindex}} + {{#_enableSearch}}{{/_enableSearch}} + {{#_enableNewTab}}{{/_enableNewTab}} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/li.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/li.tmpl.partial new file mode 100644 index 0000000000..7e2d689309 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/li.tmpl.partial @@ -0,0 +1,30 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +
      + {{#items}} + {{^dropdown}} +
    • + {{^leaf}} + + {{/leaf}} + {{#topicHref}} + {{name}} + {{/topicHref}} + {{^topicHref}} + {{{name}}} + {{/topicHref}} + {{^leaf}} + {{>partials/li}} + {{/leaf}} +
      • + {{/dropdown}} + {{#dropdown}} +
    • + +
        + {{>partials/dd-li}} +
      +
      • + {{/dropdown}} + {{/items}} +
    diff --git a/csharp/ApiDocs/_exported_templates/default/partials/logo.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/logo.tmpl.partial new file mode 100644 index 0000000000..8209615a35 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/logo.tmpl.partial @@ -0,0 +1,5 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + + + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/namespace.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/namespace.tmpl.partial new file mode 100644 index 0000000000..e8b6b141bd --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/namespace.tmpl.partial @@ -0,0 +1,13 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +

    {{>partials/title}}

    +
    {{{summary}}}
    +
    {{{conceptual}}}
    +
    {{{remarks}}}
    +{{#children}} +

    {{>partials/namespaceSubtitle}}

    + {{#children}} +

    +
    {{{summary}}}
    + {{/children}} +{{/children}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/namespaceSubtitle.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/namespaceSubtitle.tmpl.partial new file mode 100644 index 0000000000..b3a30d8c89 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/namespaceSubtitle.tmpl.partial @@ -0,0 +1,30 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} +{{^isNamespace}} +{{#inNamespace}} +{{__global.namespacesInSubtitle}} +{{/inNamespace}} +{{/isNamespace}} +{{#inClass}} +{{__global.classesInSubtitle}} +{{/inClass}} +{{#inStruct}} +{{__global.structsInSubtitle}} +{{/inStruct}} +{{#inInterface}} +{{__global.interfacesInSubtitle}} +{{/inInterface}} +{{#inEnum}} +{{__global.enumsInSubtitle}} +{{/inEnum}} +{{#inDelegate}} +{{__global.delegatesInSubtitle}} +{{/inDelegate}} +{{#inFunction}} +{{__global.functionsInSubtitle}} +{{/inFunction}} +{{#inVariable}} +{{__global.variablesInSubtitle}} +{{/inVariable}} +{{#inTypeAlias}} +{{__global.typeAliasesInSubtitle}} +{{/inTypeAlias}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/navbar.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/navbar.tmpl.partial new file mode 100644 index 0000000000..b403e0ebd4 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/navbar.tmpl.partial @@ -0,0 +1,22 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/rest.child.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/rest.child.tmpl.partial new file mode 100644 index 0000000000..b8be20305c --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/rest.child.tmpl.partial @@ -0,0 +1,87 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +{{^_disableContribution}} +{{#docurl}} + + | + Improve this Doc +{{/docurl}} +{{#sourceurl}} + + View Source +{{/sourceurl}} +{{/_disableContribution}} +

    {{operationId}}

    +{{#summary}} +
    {{{summary}}}
    +{{/summary}} +{{#description}} +
    {{{description}}}
    +{{/description}} +{{#conceptual}} +
    {{{conceptual}}}
    +{{/conceptual}} +
    Request
    +
    + 
    {{operation}} {{path}}
    +
    +{{#parameters.0}} +
    Parameters
    + + + + + + + + + + +{{/parameters.0}} +{{#parameters}} + + + + + + + {{/parameters}} + {{#parameters.0}} + +
    NameTypeValueNotes
    {{#required}}*{{/required}}{{name}}{{type}}{{default}}{{{description}}}
    +{{/parameters.0}} +{{#responses.0}} +
    +
    Responses
    + + + + + + + + + +{{/responses.0}} +{{#responses}} + + + + + + {{/responses}} + {{#responses.0}} + +
    Status CodeDescriptionSamples
    {{statusCode}}{{{description}}} + {{#examples}} +
    + Mime type: {{mimeType}} +
    + 
    {{content}}
    + {{/examples}} +
    +
    +{{/responses.0}} +{{#footer}} +
    {{{footer}}}
    +{{/footer}} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/partials/rest.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/rest.tmpl.partial new file mode 100644 index 0000000000..a7c73ecc7a --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/rest.tmpl.partial @@ -0,0 +1,37 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +

    {{name}}

    +{{#summary}} +
    {{{summary}}}
    +{{/summary}} +{{#description}} +
    {{{description}}}
    +{{/description}} +{{#conceptual}} +
    {{{conceptual}}}
    +{{/conceptual}} +{{#tags}} +

    {{name}}

    +{{#description}} +
    {{{description}}}
    +{{/description}} +{{#conceptual}} +
    {{{conceptual}}}
    +{{/conceptual}} +{{#children}} + {{>partials/rest.child}} +{{/children}} +{{/tags}} +{{!if some children are not tagged while other children are tagged, add default title}} +{{#children.0}} +{{#isTagLayout}} +

    Other APIs

    +{{/isTagLayout}} +{{/children.0}} +{{#children}} + {{>partials/rest.child}} +{{/children}} +{{#footer}} +
    {{{footer}}}
    +{{/footer}} + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/scripts.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/scripts.tmpl.partial new file mode 100644 index 0000000000..ab3204ecb9 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/scripts.tmpl.partial @@ -0,0 +1,5 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + + + + diff --git a/csharp/ApiDocs/_exported_templates/default/partials/searchResults.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/searchResults.tmpl.partial new file mode 100644 index 0000000000..26a2881f2c --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/searchResults.tmpl.partial @@ -0,0 +1,9 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +
    +
    {{__global.searchResults}}
    +
    +

    +
    +
      +
      diff --git a/csharp/ApiDocs/_exported_templates/default/partials/title.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/title.tmpl.partial new file mode 100644 index 0000000000..38c62fe55f --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/title.tmpl.partial @@ -0,0 +1,49 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} +{{#inPackage}} +Package {{name.0.value}} +{{/inPackage}} +{{#inNamespace}} +Namespace {{name.0.value}} +{{/inNamespace}} +{{#inClass}} +Class {{name.0.value}} +{{/inClass}} +{{#inStruct}} +Struct {{name.0.value}} +{{/inStruct}} +{{#inInterface}} +Interface {{name.0.value}} +{{/inInterface}} +{{#inEnum}} +Enum {{name.0.value}} +{{/inEnum}} +{{#inDelegate}} +Delegate {{name.0.value}} +{{/inDelegate}} +{{#inConstructor}} +Constructor {{name.0.value}} +{{/inConstructor}} +{{#inField}} +Field {{name.0.value}} +{{/inField}} +{{#inProperty}} +Property {{name.0.value}} +{{/inProperty}} +{{#inMethod}} +Method {{name.0.value}} +{{/inMethod}} +{{#inEvent}} +Event {{name.0.value}} +{{/inEvent}} +{{#inOperator}} +Operator {{name.0.value}} +{{/inOperator}} +{{#inEii}} +Explict Interface Implementation {{name.0.value}} +{{/inEii}} +{{#inVariable}} +Variable {{name.0.value}} +{{/inVariable}} +{{#inTypeAlias}} +Type Alias {{name.0.value}} +{{/inTypeAlias}} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/partials/toc.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/toc.tmpl.partial new file mode 100644 index 0000000000..540543fd49 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/toc.tmpl.partial @@ -0,0 +1,8 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +
      + +
      +
      +
      +
      diff --git a/csharp/ApiDocs/_exported_templates/default/partials/uref/class.header.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/uref/class.header.tmpl.partial new file mode 100644 index 0000000000..8754c71070 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/uref/class.header.tmpl.partial @@ -0,0 +1,49 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +

      {{>partials/title}}

      +
      {{{summary}}}
      +
      {{{conceptual}}}
      +{{#inheritance.0}} +
      +
      {{__global.inheritance}}
      +{{/inheritance.0}} +{{#inheritance.0.value}} + {{>partials/uref/inheritance}} +{{/inheritance.0.value}} +{{#inheritance.0}} +
      {{name.0.value}}
      +
      +{{/inheritance.0}} +{{#inheritedMembers.0}} +
      +
      {{__global.inheritedMembers}}
      +{{/inheritedMembers.0}} +{{#inheritedMembers}} +
      + {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
      +{{/inheritedMembers}} +{{#inheritedMembers.0}} +
      +{{/inheritedMembers.0}} +{{#namespace.0}} +
      {{__global.namespace}}: {{{value.specName.0.value}}}
      +{{/namespace.0}} +{{#package.0}} +
      {{__global.package}}: {{{value.specName.0.value}}}
      +{{/package.0}} +{{#remarks}} +
      {{__global.remarks}}
      +
      {{{remarks}}}
      +{{/remarks}} +{{#example.0}} +
      {{__global.examples}}
      +{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/partials/uref/class.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/uref/class.tmpl.partial new file mode 100644 index 0000000000..f4eb1194f2 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/uref/class.tmpl.partial @@ -0,0 +1,269 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +{{>partials/uref/class.header}} +{{#children}} +

      {{>partials/classSubtitle}}

      +{{#children}} +{{^_disableContribution}} +{{#docurl}} + + | + {{__global.improveThisDoc}} +{{/docurl}} +{{#sourceurl}} + + {{__global.viewSource}} +{{/sourceurl}} +{{/_disableContribution}} +{{#overload}} + +{{/overload}} +

      {{name.0.value}}

      +
      {{{summary}}}
      +
      {{{conceptual}}}
      +
      {{__global.declaration}}
      +{{#syntax}} +
      + 
      {{syntax.content.0.value}}
      +
      +{{#parameters.0}} +
      {{__global.parameters}}
      + + + + + + + + + +{{/parameters.0}} +{{#parameters}} + + + + + +{{/parameters}} +{{#parameters.0}} + +
      {{__global.type}}{{__global.name}}{{__global.description}}
      {{{type.specName.0.value}}}{{{id}}} + {{{description}}} + {{>partials/uref/parameters}} +
      +{{/parameters.0}} +{{#return}} +
      {{__global.returns}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      +{{/return}} +{{#typeParameters.0}} +
      {{__global.typeParameters}}
      + + + + + + + + +{{/typeParameters.0}} +{{#typeParameters}} + + + + +{{/typeParameters}} +{{#typeParameters.0}} + +
      {{__global.name}}{{__global.description}}
      {{{id}}}{{{description}}}
      +{{/typeParameters.0}} +{{#fieldValue}} +
      {{__global.fieldValue}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      +{{/fieldValue}} +{{#propertyValue}} +
      {{__global.propertyValue}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      +{{/propertyValue}} +{{#eventType}} +
      {{__global.eventType}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{type.specName.0.value}}}{{{description}}}
      +{{/eventType}} +{{#variableValue}} +
      {{__global.variableValue}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      +{{/variableValue}} +{{#typeAliasType}} +
      {{__global.typeAliasType}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      +{{/typeAliasType}} +{{/syntax}} +{{#overridden}} +
      {{__global.overrides}}
      +
      +{{/overridden}} +{{#implements.0}} +
      {{__global.implements}}
      +{{/implements.0}} +{{#implements}} + {{#definition}} +
      + {{/definition}} + {{^definition}} +
      + {{/definition}} +{{/implements}} +{{#remarks}} +
      {{__global.remarks}}
      +
      {{{remarks}}}
      +{{/remarks}} +{{#example.0}} +
      {{__global.examples}}
      +{{/example.0}} +{{#example}} +{{{.}}} +{{/example}} +{{#exceptions.0}} +
      {{__global.exceptions}}
      + + + + + + + + +{{/exceptions.0}} +{{#exceptions.0.value}} + + + + +{{/exceptions.0.value}} +{{#exceptions.0}} + +
      {{__global.type}}{{__global.condition}}
      {{{type.specName.0.value}}}{{{description}}}
      +{{/exceptions.0}} +{{#seealso.0}} +
      {{__global.seealso}}
      +
      +{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
      {{{type.specName.0.value}}}
      + {{/isCref}} + {{^isCref}} +
      {{{url}}}
      + {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
      +{{/seealso.0}} +{{/children}} +{{/children}} +{{#extensionMethods.0}} +

      {{__global.extensionMethods}}

      +{{/extensionMethods.0}} +{{#extensionMethods}} +
      + {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
      +{{/extensionMethods}} +{{#seealso.0}} +

      {{__global.seealso}}

      +
      +{{/seealso.0}} +{{#seealso}} + {{#isCref}} +
      {{{type.specName.0.value}}}
      + {{/isCref}} + {{^isCref}} +
      {{{url}}}
      + {{/isCref}} +{{/seealso}} +{{#seealso.0}} +
      +{{/seealso.0}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/uref/enum.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/uref/enum.tmpl.partial new file mode 100644 index 0000000000..ef9fcb3195 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/uref/enum.tmpl.partial @@ -0,0 +1,35 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +{{>partials/uref/class.header}} +{{#children}} +

      {{>partials/classSubtitle}}

      + + + + + + + + + {{#children}} + + + + + {{/children}} + +
      {{__global.name}}{{__global.description}}
      {{name.0.value}}{{{summary}}}
      +{{/children}} +{{#extensionMethods.0}} +

      {{__global.extensionMethods}}

      +{{/extensionMethods.0}} +{{#extensionMethods}} +
      + {{#definition}} + + {{/definition}} + {{^definition}} + + {{/definition}} +
      +{{/extensionMethods}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/uref/inheritance.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/uref/inheritance.tmpl.partial new file mode 100644 index 0000000000..7e102fed00 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/uref/inheritance.tmpl.partial @@ -0,0 +1,4 @@ +{{#inheritance}} + {{>partials/uref/inheritance}} +{{/inheritance}} +
      {{{type.specName.0.value}}}
      diff --git a/csharp/ApiDocs/_exported_templates/default/partials/uref/namespace.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/uref/namespace.tmpl.partial new file mode 100644 index 0000000000..589100907a --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/uref/namespace.tmpl.partial @@ -0,0 +1,221 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +

      {{>partials/title}}

      +
      {{{summary}}}
      +
      {{{conceptual}}}
      +{{#package.0}} +
      {{__global.package}}: {{{value.specName.0.value}}}
      +{{/package.0}} +
      {{{remarks}}}
      +{{#children}} +

      {{>partials/namespaceSubtitle}}

      + {{^isEmbedded}} + {{#children}} +

      +
      {{{summary}}}
      + {{/children}} + {{/isEmbedded}} + {{#isEmbedded}} + {{#children}} + {{^_disableContribution}} + {{#docurl}} + + | + {{__global.improveThisDoc}} + {{/docurl}} + {{#sourceurl}} + + {{__global.viewSource}} + {{/sourceurl}} + {{/_disableContribution}} + {{#overload}} + + {{/overload}} +

      {{name.0.value}}

      +
      {{{summary}}}
      +
      {{{conceptual}}}
      +
      {{__global.declaration}}
      + {{#syntax}} +
      + 
      {{syntax.content.0.value}}
      +
      + {{#parameters.0}} +
      {{__global.parameters}}
      + + + + + + + + + + {{/parameters.0}} + {{#parameters}} + + + + + + {{/parameters}} + {{#parameters.0}} + +
      {{__global.type}}{{__global.name}}{{__global.description}}
      {{{type.specName.0.value}}}{{{id}}} + {{{description}}} + {{>partials/uref/parameters}} +
      + {{/parameters.0}} + {{#return}} +
      {{__global.returns}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      + {{/return}} + {{#typeParameters.0}} +
      {{__global.typeParameters}}
      + + + + + + + + + {{/typeParameters.0}} + {{#typeParameters}} + + + + + {{/typeParameters}} + {{#typeParameters.0}} + +
      {{__global.name}}{{__global.description}}
      {{{id}}}{{{description}}}
      + {{/typeParameters.0}} + {{#fieldValue}} +
      {{__global.fieldValue}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      + {{/fieldValue}} + {{#propertyValue}} +
      {{__global.propertyValue}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{value.type.0.specName.0.value}}}{{{value.description}}}
      + {{/propertyValue}} + {{#eventType}} +
      {{__global.eventType}}
      + + + + + + + + + + + + + +
      {{__global.type}}{{__global.description}}
      {{{type.specName.0.value}}}{{{description}}}
      + {{/eventType}} + {{/syntax}} + {{#overridden}} +
      {{__global.overrides}}
      +
      + {{/overridden}} + {{#implements.0}} +
      {{__global.implements}}
      + {{/implements.0}} + {{#implements}} + {{#definition}} +
      + {{/definition}} + {{^definition}} +
      + {{/definition}} + {{/implements}} + {{#remarks}} +
      {{__global.remarks}}
      +
      {{{remarks}}}
      + {{/remarks}} + {{#example.0}} +
      {{__global.examples}}
      + {{/example.0}} + {{#example}} + {{{.}}} + {{/example}} + {{#exceptions.0}} +
      {{__global.exceptions}}
      + + + + + + + + + {{/exceptions.0}} + {{#exceptions.0.value}} + + + + + {{/exceptions.0.value}} + {{#exceptions.0}} + +
      {{__global.type}}{{__global.condition}}
      {{{type.specName.0.value}}}{{{description}}}
      + {{/exceptions.0}} + {{#seealso.0}} +
      {{__global.seealso}}
      +
      + {{/seealso.0}} + {{#seealso}} + {{#isCref}} +
      {{{type.specName.0.value}}}
      + {{/isCref}} + {{^isCref}} +
      {{{url}}}
      + {{/isCref}} + {{/seealso}} + {{#seealso.0}} +
      + {{/seealso.0}} + {{/children}} + {{/isEmbedded}} +{{/children}} diff --git a/csharp/ApiDocs/_exported_templates/default/partials/uref/parameters.tmpl.partial b/csharp/ApiDocs/_exported_templates/default/partials/uref/parameters.tmpl.partial new file mode 100644 index 0000000000..25feab9c51 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/partials/uref/parameters.tmpl.partial @@ -0,0 +1,28 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +{{#properties.0}} +
      Properties
      + + + + + + + + + +{{/properties.0}} +{{#properties}} + + + + + +{{/properties}} +{{#properties.0}} + +
      {{__global.type}}{{__global.name}}{{__global.description}}
      {{{type.specName.0.value}}}{{{id}}} + {{{description}}} + {{>partials/parameters}} +
      +{{/properties.0}} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/search-stopwords.json b/csharp/ApiDocs/_exported_templates/default/search-stopwords.json new file mode 100644 index 0000000000..0bdcc2c004 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/search-stopwords.json @@ -0,0 +1,121 @@ +[ + "a", + "able", + "about", + "across", + "after", + "all", + "almost", + "also", + "am", + "among", + "an", + "and", + "any", + "are", + "as", + "at", + "be", + "because", + "been", + "but", + "by", + "can", + "cannot", + "could", + "dear", + "did", + "do", + "does", + "either", + "else", + "ever", + "every", + "for", + "from", + "get", + "got", + "had", + "has", + "have", + "he", + "her", + "hers", + "him", + "his", + "how", + "however", + "i", + "if", + "in", + "into", + "is", + "it", + "its", + "just", + "least", + "let", + "like", + "likely", + "may", + "me", + "might", + "most", + "must", + "my", + "neither", + "no", + "nor", + "not", + "of", + "off", + "often", + "on", + "only", + "or", + "other", + "our", + "own", + "rather", + "said", + "say", + "says", + "she", + "should", + "since", + "so", + "some", + "than", + "that", + "the", + "their", + "them", + "then", + "there", + "these", + "they", + "this", + "tis", + "to", + "too", + "twas", + "us", + "wants", + "was", + "we", + "were", + "what", + "when", + "where", + "which", + "while", + "who", + "whom", + "why", + "will", + "with", + "would", + "yet", + "you", + "your" +] diff --git a/csharp/ApiDocs/_exported_templates/default/styles/docfx.css b/csharp/ApiDocs/_exported_templates/default/styles/docfx.css new file mode 100644 index 0000000000..64dcde3385 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/docfx.css @@ -0,0 +1,1032 @@ +/* Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the MIT License. See License.txt in the project root for license information. */ +html, +body { + font-family: 'Segoe UI', Tahoma, Helvetica, sans-serif; + height: 100%; +} +button, +a { + color: #337ab7; + cursor: pointer; +} +button:hover, +button:focus, +a:hover, +a:focus { + color: #23527c; + text-decoration: none; +} +a.disable, +a.disable:hover { + text-decoration: none; + cursor: default; + color: #000000; +} + +h1, h2, h3, h4, h5, h6, .text-break { + word-wrap: break-word; + word-break: break-word; +} + +h1 mark, +h2 mark, +h3 mark, +h4 mark, +h5 mark, +h6 mark { + padding: 0; +} + +.inheritance .level0:before, +.inheritance .level1:before, +.inheritance .level2:before, +.inheritance .level3:before, +.inheritance .level4:before, +.inheritance .level5:before, +.inheritance .level6:before, +.inheritance .level7:before, +.inheritance .level8:before, +.inheritance .level9:before { + content: '↳'; + margin-right: 5px; +} + +.inheritance .level0 { + margin-left: 0em; +} + +.inheritance .level1 { + margin-left: 1em; +} + +.inheritance .level2 { + margin-left: 2em; +} + +.inheritance .level3 { + margin-left: 3em; +} + +.inheritance .level4 { + margin-left: 4em; +} + +.inheritance .level5 { + margin-left: 5em; +} + +.inheritance .level6 { + margin-left: 6em; +} + +.inheritance .level7 { + margin-left: 7em; +} + +.inheritance .level8 { + margin-left: 8em; +} + +.inheritance .level9 { + margin-left: 9em; +} + +.level0.summary { + margin: 2em 0 2em 0; +} + +.level1.summary { + margin: 1em 0 1em 0; +} + +span.parametername, +span.paramref, +span.typeparamref { + font-style: italic; +} +span.languagekeyword{ + font-weight: bold; +} + +svg:hover path { + fill: #ffffff; +} + +.hljs { + display: inline; + background-color: inherit; + padding: 0; +} +/* additional spacing fixes */ +.btn + .btn { + margin-left: 10px; +} +.btn.pull-right { + margin-left: 10px; + margin-top: 5px; +} +.table { + margin-bottom: 10px; +} +table p { + margin-bottom: 0; +} +table a { + display: inline-block; +} + +/* Make hidden attribute compatible with old browser.*/ +[hidden] { + display: none !important; +} + +h1, +.h1, +h2, +.h2, +h3, +.h3 { + margin-top: 15px; + margin-bottom: 10px; + font-weight: 400; +} +h4, +.h4, +h5, +.h5, +h6, +.h6 { + margin-top: 10px; + margin-bottom: 5px; +} +.navbar { + margin-bottom: 0; +} +#wrapper { + min-height: 100%; + position: relative; +} +/* blends header footer and content together with gradient effect */ +.grad-top { + /* For Safari 5.1 to 6.0 */ + /* For Opera 11.1 to 12.0 */ + /* For Firefox 3.6 to 15 */ + background: linear-gradient(rgba(0, 0, 0, 0.05), rgba(0, 0, 0, 0)); + /* Standard syntax */ + height: 5px; +} +.grad-bottom { + /* For Safari 5.1 to 6.0 */ + /* For Opera 11.1 to 12.0 */ + /* For Firefox 3.6 to 15 */ + background: linear-gradient(rgba(0, 0, 0, 0), rgba(0, 0, 0, 0.05)); + /* Standard syntax */ + height: 5px; +} +.divider { + margin: 0 5px; + color: #cccccc; +} +hr { + border-color: #cccccc; +} +header { + position: fixed; + top: 0; + left: 0; + right: 0; + z-index: 1000; +} +header .navbar { + border-width: 0 0 1px; + border-radius: 0; +} +.navbar-brand { + font-size: inherit; + padding: 0; +} +.navbar-collapse { + margin: 0 -15px; +} +.subnav { + min-height: 40px; +} + +.inheritance h5, .inheritedMembers h5{ + padding-bottom: 5px; + border-bottom: 1px solid #ccc; +} + +article h1, article h2, article h3, article h4{ + margin-top: 25px; +} + +article h4{ + border: 0; + font-weight: bold; + margin-top: 2em; +} + +article span.small.pull-right{ + margin-top: 20px; +} + +article section { + margin-left: 1em; +} + +/*.expand-all { + padding: 10px 0; +}*/ +.breadcrumb { + margin: 0; + padding: 10px 0; + background-color: inherit; + white-space: nowrap; +} +.breadcrumb > li + li:before { + content: "\00a0/"; +} +#autocollapse.collapsed .navbar-header { + float: none; +} +#autocollapse.collapsed .navbar-toggle { + display: block; +} +#autocollapse.collapsed .navbar-collapse { + border-top: 1px solid transparent; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1); +} +#autocollapse.collapsed .navbar-collapse.collapse { + display: none !important; +} +#autocollapse.collapsed .navbar-nav { + float: none !important; + margin: 7.5px -15px; +} +#autocollapse.collapsed .navbar-nav > li { + float: none; +} +#autocollapse.collapsed .navbar-nav > li > a { + padding-top: 10px; + padding-bottom: 10px; +} +#autocollapse.collapsed .collapse.in, +#autocollapse.collapsed .collapsing { + display: block !important; +} +#autocollapse.collapsed .collapse.in .navbar-right, +#autocollapse.collapsed .collapsing .navbar-right { + float: none !important; +} +#autocollapse .form-group { + width: 100%; +} +#autocollapse .form-control { + width: 100%; +} +#autocollapse .navbar-header { + margin-left: 0; + margin-right: 0; +} +#autocollapse .navbar-brand { + margin-left: 0; +} +.collapse.in, +.collapsing { + text-align: center; +} +.collapsing .navbar-form { + margin: 0 auto; + max-width: 400px; + padding: 10px 15px; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); +} +.collapsed .collapse.in .navbar-form { + margin: 0 auto; + max-width: 400px; + padding: 10px 15px; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); +} +.navbar .navbar-nav { + display: inline-block; +} +.docs-search { + background: white; + vertical-align: middle; +} +.docs-search > .search-query { + font-size: 14px; + border: 0; + width: 120%; + color: #555; +} +.docs-search > .search-query:focus { + outline: 0; +} +.search-results-frame { + clear: both; + display: table; + width: 100%; +} +.search-results.ng-hide { + display: none; +} +.search-results-container { + padding-bottom: 1em; + border-top: 1px solid #111; + background: rgba(25, 25, 25, 0.5); +} +.search-results-container .search-results-group { + padding-top: 50px !important; + padding: 10px; +} +.search-results-group-heading { + font-family: "Open Sans"; + padding-left: 10px; + color: white; +} +.search-close { + position: absolute; + left: 50%; + margin-left: -100px; + color: white; + text-align: center; + padding: 5px; + background: #333; + border-top-right-radius: 5px; + border-top-left-radius: 5px; + width: 200px; + box-shadow: 0 0 10px #111; +} +#search { + display: none; +} + +/* Search results display*/ +#search-results { + max-width: 960px !important; + margin-top: 120px; + margin-bottom: 115px; + margin-left: auto; + margin-right: auto; + line-height: 1.8; + display: none; +} + +#search-results>.search-list { + text-align: center; + font-size: 2.5rem; + margin-bottom: 50px; +} + +#search-results p { + text-align: center; +} + +#search-results p .index-loading { + animation: index-loading 1.5s infinite linear; + -webkit-animation: index-loading 1.5s infinite linear; + -o-animation: index-loading 1.5s infinite linear; + font-size: 2.5rem; +} + +@keyframes index-loading { + from { transform: scale(1) rotate(0deg);} + to { transform: scale(1) rotate(360deg);} +} + +@-webkit-keyframes index-loading { + from { -webkit-transform: rotate(0deg);} + to { -webkit-transform: rotate(360deg);} +} + +@-o-keyframes index-loading { + from { -o-transform: rotate(0deg);} + to { -o-transform: rotate(360deg);} +} + +#search-results .sr-items { + font-size: 24px; +} + +.sr-item { + margin-bottom: 25px; +} + +.sr-item>.item-href { + font-size: 14px; + color: #093; +} + +.sr-item>.item-brief { + font-size: 13px; +} + +.pagination>li>a { + color: #47A7A0 +} + +.pagination>.active>a { + background-color: #47A7A0; + border-color: #47A7A0; +} + +.fixed_header { + position: fixed; + width: 100%; + padding-bottom: 10px; + padding-top: 10px; + margin: 0px; + top: 0; + z-index: 9999; + left: 0; +} + +.fixed_header+.toc{ + margin-top: 50px; + margin-left: 0; +} + +.sidenav, .fixed_header, .toc { + background-color: #f1f1f1; +} + +.sidetoc { + position: fixed; + width: 260px; + top: 150px; + bottom: 0; + overflow-x: hidden; + overflow-y: auto; + background-color: #f1f1f1; + border-left: 1px solid #e7e7e7; + border-right: 1px solid #e7e7e7; + z-index: 1; +} + +.sidetoc.shiftup { + bottom: 70px; +} + +body .toc{ + background-color: #f1f1f1; + overflow-x: hidden; +} + +.sidetoggle.ng-hide { + display: block !important; +} +.sidetoc-expand > .caret { + margin-left: 0px; + margin-top: -2px; +} +.sidetoc-expand > .caret-side { + border-left: 4px solid; + border-top: 4px solid transparent; + border-bottom: 4px solid transparent; + margin-left: 4px; + margin-top: -4px; +} +.sidetoc-heading { + font-weight: 500; +} + +.toc { + margin: 0px 0 0 10px; + padding: 0 10px; +} +.expand-stub { + position: absolute; + left: -10px; +} +.toc .nav > li > a.sidetoc-expand { + position: absolute; + top: 0; + left: 0; +} +.toc .nav > li > a { + color: #666666; + margin-left: 5px; + display: block; + padding: 0; +} +.toc .nav > li > a:hover, +.toc .nav > li > a:focus { + color: #000000; + background: none; + text-decoration: inherit; +} +.toc .nav > li.active > a { + color: #337ab7; +} +.toc .nav > li.active > a:hover, +.toc .nav > li.active > a:focus { + color: #23527c; +} + +.toc .nav > li> .expand-stub { + cursor: pointer; +} + +.toc .nav > li.active > .expand-stub::before, +.toc .nav > li.in > .expand-stub::before, +.toc .nav > li.in.active > .expand-stub::before, +.toc .nav > li.filtered > .expand-stub::before { + content: "-"; +} + +.toc .nav > li > .expand-stub::before, +.toc .nav > li.active > .expand-stub::before { + content: "+"; +} + +.toc .nav > li.filtered > ul, +.toc .nav > li.in > ul { + display: block; +} + +.toc .nav > li > ul { + display: none; +} + +.toc ul{ + font-size: 12px; + margin: 0 0 0 3px; +} + +.toc .level1 > li { + font-weight: bold; + margin-top: 10px; + position: relative; + font-size: 16px; +} +.toc .level2 { + font-weight: normal; + margin: 5px 0 0 15px; + font-size: 14px; +} +.toc-toggle { + display: none; + margin: 0 15px 0px 15px; +} +.sidefilter { + position: fixed; + top: 90px; + width: 260px; + background-color: #f1f1f1; + padding: 15px; + border-left: 1px solid #e7e7e7; + border-right: 1px solid #e7e7e7; + z-index: 1; +} +.toc-filter { + border-radius: 5px; + background: #fff; + color: #666666; + padding: 5px; + position: relative; + margin: 0 5px 0 5px; +} +.toc-filter > input { + border: 0; + color: #666666; + padding-left: 20px; + padding-right: 20px; + width: 100%; +} +.toc-filter > input:focus { + outline: 0; +} +.toc-filter > .filter-icon { + position: absolute; + top: 10px; + left: 5px; +} +.toc-filter > .clear-icon { + position: absolute; + top: 10px; + right: 5px; +} +.article { + margin-top: 120px; + margin-bottom: 115px; +} + +#_content>a{ + margin-top: 105px; +} + +.article.grid-right { + margin-left: 280px; +} + +.inheritance hr { + margin-top: 5px; + margin-bottom: 5px; +} +.article img { + max-width: 100%; +} +.sideaffix { + margin-top: 50px; + font-size: 12px; + max-height: 100%; + overflow: hidden; + top: 100px; + bottom: 10px; + position: fixed; +} +.sideaffix.shiftup { + bottom: 70px; +} +.affix { + position: relative; + height: 100%; +} +.sideaffix > div.contribution { + margin-bottom: 20px; +} +.sideaffix > div.contribution > ul > li > a.contribution-link { + padding: 6px 10px; + font-weight: bold; + font-size: 14px; +} +.sideaffix > div.contribution > ul > li > a.contribution-link:hover { + background-color: #ffffff; +} +.sideaffix ul.nav > li > a:focus { + background: none; +} +.affix h5 { + font-weight: bold; + text-transform: uppercase; + padding-left: 10px; + font-size: 12px; +} +.affix > ul.level1 { + overflow: hidden; + padding-bottom: 10px; + height: calc(100% - 100px); +} +.affix ul > li > a:before { + color: #cccccc; + position: absolute; +} +.affix ul > li > a:hover { + background: none; + color: #666666; +} +.affix ul > li.active > a, +.affix ul > li.active > a:before { + color: #337ab7; +} +.affix ul > li > a { + padding: 5px 12px; + color: #666666; +} +.affix > ul > li.active:last-child { + margin-bottom: 50px; +} +.affix > ul > li > a:before { + content: "|"; + font-size: 16px; + top: 1px; + left: 0; +} +.affix > ul > li.active > a, +.affix > ul > li.active > a:before { + color: #337ab7; + font-weight: bold; +} +.affix ul ul > li > a { + padding: 2px 15px; +} +.affix ul ul > li > a:before { + content: ">"; + font-size: 14px; + top: -1px; + left: 5px; +} +.affix ul > li > a:before, +.affix ul ul { + display: none; +} +.affix ul > li.active > ul, +.affix ul > li.active > a:before, +.affix ul > li > a:hover:before { + display: block; + white-space: nowrap; +} +.codewrapper { + position: relative; +} +.trydiv { + height: 0px; +} +.tryspan { + position: absolute; + top: 0px; + right: 0px; + border-style: solid; + border-radius: 0px 4px; + box-sizing: border-box; + border-width: 1px; + border-color: #cccccc; + text-align: center; + padding: 2px 8px; + background-color: white; + font-size: 12px; + cursor: pointer; + z-index: 100; + display: none; + color: #767676; +} +.tryspan:hover { + background-color: #3b8bd0; + color: white; + border-color: #3b8bd0; +} +.codewrapper:hover .tryspan { + display: block; +} +.sample-response .response-content{ + max-height: 200px; +} +footer { + position: absolute; + left: 0; + right: 0; + bottom: 0; + z-index: 1000; +} +.footer { + border-top: 1px solid #e7e7e7; + background-color: #f8f8f8; + padding: 15px 0; +} +@media (min-width: 768px) { + #sidetoggle.collapse { + display: block; + } + .topnav .navbar-nav { + float: none; + white-space: nowrap; + } + .topnav .navbar-nav > li { + float: none; + display: inline-block; + } +} +@media only screen and (max-width: 768px) { + #mobile-indicator { + display: block; + } + /* TOC display for responsive */ + .article { + margin-top: 30px !important; + } + header { + position: static; + } + .topnav { + text-align: center; + } + .sidenav { + padding: 15px 0; + margin-left: -15px; + margin-right: -15px; + } + .sidefilter { + position: static; + width: auto; + float: none; + border: none; + } + .sidetoc { + position: static; + width: auto; + float: none; + padding-bottom: 0px; + border: none; + } + .toc .nav > li, .toc .nav > li >a { + display: inline-block; + } + .toc li:after { + margin-left: -3px; + margin-right: 5px; + content: ", "; + color: #666666; + } + .toc .level1 > li { + display: block; + } + + .toc .level1 > li:after { + display: none; + } + .article.grid-right { + margin-left: 0; + } + .grad-top, + .grad-bottom { + display: none; + } + .toc-toggle { + display: block; + } + .sidetoggle.ng-hide { + display: none !important; + } + /*.expand-all { + display: none; + }*/ + .sideaffix { + display: none; + } + .mobile-hide { + display: none; + } + .breadcrumb { + white-space: inherit; + } + + /* workaround for #hashtag url is no longer needed*/ + h1:before, + h2:before, + h3:before, + h4:before { + content: ''; + display: none; + } +} + +/* For toc iframe */ +@media (max-width: 260px) { + .toc .level2 > li { + display: block; + } + + .toc .level2 > li:after { + display: none; + } +} + +/* Code snippet */ +code { + color: #717374; + background-color: #f1f2f3; +} + +a code { + color: #337ab7; + background-color: #f1f2f3; +} + +a code:hover { + text-decoration: underline; +} + +.hljs-keyword { + color: rgb(86,156,214); +} + +.hljs-string { + color: rgb(214, 157, 133); +} + +pre { + border: 0; +} + +/* For code snippet line highlight */ +pre > code .line-highlight { + background-color: #ffffcc; +} + +/* Alerts */ +.alert h5 { + text-transform: uppercase; + font-weight: bold; + margin-top: 0; +} + +.alert h5:before { + position:relative; + top:1px; + display:inline-block; + font-family:'Glyphicons Halflings'; + line-height:1; + -webkit-font-smoothing:antialiased; + -moz-osx-font-smoothing:grayscale; + margin-right: 5px; + font-weight: normal; +} + +.alert-info h5:before { + content:"\e086" +} + +.alert-warning h5:before { + content:"\e127" +} + +.alert-danger h5:before { + content:"\e107" +} + +/* For Embedded Video */ +div.embeddedvideo { + padding-top: 56.25%; + position: relative; + width: 100%; +} + +div.embeddedvideo iframe { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + width: 100%; + height: 100%; +} + +/* For printer */ +@media print{ + .article.grid-right { + margin-top: 0px; + margin-left: 0px; + } + .sideaffix { + display: none; + } + .mobile-hide { + display: none; + } + .footer { + display: none; + } +} + +/* For tabbed content */ + +.tabGroup { + margin-top: 1rem; } + .tabGroup ul[role="tablist"] { + margin: 0; + padding: 0; + list-style: none; } + .tabGroup ul[role="tablist"] > li { + list-style: none; + display: inline-block; } + .tabGroup a[role="tab"] { + color: #6e6e6e; + box-sizing: border-box; + display: inline-block; + padding: 5px 7.5px; + text-decoration: none; + border-bottom: 2px solid #fff; } + .tabGroup a[role="tab"]:hover, .tabGroup a[role="tab"]:focus, .tabGroup a[role="tab"][aria-selected="true"] { + border-bottom: 2px solid #0050C5; } + .tabGroup a[role="tab"][aria-selected="true"] { + color: #222; } + .tabGroup a[role="tab"]:hover, .tabGroup a[role="tab"]:focus { + color: #0050C5; } + .tabGroup a[role="tab"]:focus { + outline: 1px solid #0050C5; + outline-offset: -1px; } + @media (min-width: 768px) { + .tabGroup a[role="tab"] { + padding: 5px 15px; } } + .tabGroup section[role="tabpanel"] { + border: 1px solid #e0e0e0; + padding: 15px; + margin: 0; + overflow: hidden; } + .tabGroup section[role="tabpanel"] > .codeHeader, + .tabGroup section[role="tabpanel"] > pre { + margin-left: -16px; + margin-right: -16px; } + .tabGroup section[role="tabpanel"] > :first-child { + margin-top: 0; } + .tabGroup section[role="tabpanel"] > pre:last-child { + display: block; + margin-bottom: -16px; } + +.mainContainer[dir='rtl'] main ul[role="tablist"] { + margin: 0; } + +/* Color theme */ + +/* These are not important, tune down **/ +.decalaration, .fieldValue, .parameters, .returns { + color: #a2a2a2; +} + +/* Major sections, increase visibility **/ +#fields, #properties, #methods, #events { + font-weight: bold; + margin-top: 2em; +} diff --git a/csharp/ApiDocs/_exported_templates/default/styles/docfx.js b/csharp/ApiDocs/_exported_templates/default/styles/docfx.js new file mode 100644 index 0000000000..b6167dbd61 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/docfx.js @@ -0,0 +1,1160 @@ +// Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information. +$(function () { + var active = 'active'; + var expanded = 'in'; + var collapsed = 'collapsed'; + var filtered = 'filtered'; + var show = 'show'; + var hide = 'hide'; + var util = new utility(); + + workAroundFixedHeaderForAnchors(); + highlight(); + enableSearch(); + + renderTables(); + renderAlerts(); + renderLinks(); + renderNavbar(); + renderSidebar(); + renderAffix(); + renderFooter(); + renderLogo(); + + breakText(); + renderTabs(); + + window.refresh = function (article) { + // Update markup result + if (typeof article == 'undefined' || typeof article.content == 'undefined') + console.error("Null Argument"); + $("article.content").html(article.content); + + highlight(); + renderTables(); + renderAlerts(); + renderAffix(); + renderTabs(); + } + + // Add this event listener when needed + // window.addEventListener('content-update', contentUpdate); + + function breakText() { + $(".xref").addClass("text-break"); + var texts = $(".text-break"); + texts.each(function () { + $(this).breakWord(); + }); + } + + // Styling for tables in conceptual documents using Bootstrap. + // See http://getbootstrap.com/css/#tables + function renderTables() { + $('table').addClass('table table-bordered table-striped table-condensed').wrap('
      '); + } + + // Styling for alerts. + function renderAlerts() { + $('.NOTE, .TIP').addClass('alert alert-info'); + $('.WARNING').addClass('alert alert-warning'); + $('.IMPORTANT, .CAUTION').addClass('alert alert-danger'); + } + + // Enable anchors for headings. + (function () { + anchors.options = { + placement: 'left', + visible: 'hover' + }; + anchors.add('article h2:not(.no-anchor), article h3:not(.no-anchor), article h4:not(.no-anchor)'); + })(); + + // Open links to different host in a new window. + function renderLinks() { + if ($("meta[property='docfx:newtab']").attr("content") === "true") { + $(document.links).filter(function () { + return this.hostname !== window.location.hostname; + }).attr('target', '_blank'); + } + } + + // Enable highlight.js + function highlight() { + $('pre code').each(function (i, block) { + hljs.highlightBlock(block); + }); + $('pre code[highlight-lines]').each(function (i, block) { + if (block.innerHTML === "") return; + var lines = block.innerHTML.split('\n'); + + let queryString = block.getAttribute('highlight-lines'); + if (!queryString) return; + + var ranges = queryString.split(','); + for (var j = 0, range; range = ranges[j++];) { + var found = range.match(/^(\d+)\-(\d+)?$/); + if (found) { + // consider region as `{startlinenumber}-{endlinenumber}`, in which {endlinenumber} is optional + var start = +found[1]; + var end = +found[2]; + if (isNaN(end) || end > lines.length) { + end = lines.length; + } + } else { + // consider region as a sigine line number + if (isNaN(range)) continue; + var start = +range; + var end = start; + } + if (start <= 0 || end <= 0 || start > end || start > lines.length) { + // skip current region if invalid + continue; + } + lines[start - 1] = '' + lines[start - 1]; + lines[end - 1] = lines[end - 1] + ''; + } + + block.innerHTML = lines.join('\n'); + }); + } + + // Support full-text-search + function enableSearch() { + var query; + var relHref = $("meta[property='docfx\\:rel']").attr("content"); + if (typeof relHref === 'undefined') { + return; + } + try { + var worker = new Worker(relHref + 'styles/search-worker.js'); + webWorkerSearch(); + + renderSearchBox(); + highlightKeywords(); + addSearchEvent(); + } catch (e) { + console.error(e); + } + + //Adjust the position of search box in navbar + function renderSearchBox() { + autoCollapse(); + $(window).on('resize', autoCollapse); + $(document).on('click', '.navbar-collapse.in', function (e) { + if ($(e.target).is('a')) { + $(this).collapse('hide'); + } + }); + + function autoCollapse() { + var navbar = $('#autocollapse'); + if (navbar.height() === null) { + setTimeout(autoCollapse, 300); + } + navbar.removeClass(collapsed); + if (navbar.height() > 60) { + navbar.addClass(collapsed); + } + } + } + + function webWorkerSearch() { + console.log("using Web Worker"); + var indexReady = $.Deferred(); + + worker.onmessage = function (oEvent) { + switch (oEvent.data.e) { + case 'index-ready': + indexReady.resolve(); + break; + case 'query-ready': + var hits = oEvent.data.d; + handleSearchResults(hits); + break; + } + } + + indexReady.promise().done(function () { + $("body").bind("queryReady", function () { + worker.postMessage({ q: query }); + }); + if (query && (query.length >= 3)) { + worker.postMessage({ q: query }); + } + }); + } + + // Highlight the searching keywords + function highlightKeywords() { + var q = url('?q'); + if (q) { + var keywords = q.split("%20"); + keywords.forEach(function (keyword) { + if (keyword !== "") { + $('.data-searchable *').mark(keyword); + $('article *').mark(keyword); + } + }); + } + } + + function addSearchEvent() { + $('body').bind("searchEvent", function () { + $('#search-query').keypress(function (e) { + return e.which !== 13; + }); + + $('#search-query').keyup(function () { + query = $(this).val(); + if (query.length < 3) { + flipContents("show"); + } else { + flipContents("hide"); + $("body").trigger("queryReady"); + $('#search-results>.search-list>span').text('"' + query + '"'); + } + }).off("keydown"); + }); + } + + function flipContents(action) { + if (action === "show") { + $('.hide-when-search').show(); + $('#search-results').hide(); + } else { + $('.hide-when-search').hide(); + $('#search-results').show(); + } + } + + function relativeUrlToAbsoluteUrl(currentUrl, relativeUrl) { + var currentItems = currentUrl.split(/\/+/); + var relativeItems = relativeUrl.split(/\/+/); + var depth = currentItems.length - 1; + var items = []; + for (var i = 0; i < relativeItems.length; i++) { + if (relativeItems[i] === '..') { + depth--; + } else if (relativeItems[i] !== '.') { + items.push(relativeItems[i]); + } + } + return currentItems.slice(0, depth).concat(items).join('/'); + } + + function extractContentBrief(content) { + var briefOffset = 512; + var words = query.split(/\s+/g); + var queryIndex = content.indexOf(words[0]); + if (queryIndex > briefOffset) { + return "..." + content.slice(queryIndex - briefOffset, queryIndex + briefOffset) + "..."; + } else if (queryIndex <= briefOffset) { + return content.slice(0, queryIndex + briefOffset) + "..."; + } + } + + function handleSearchResults(hits) { + var numPerPage = 10; + var pagination = $('#pagination'); + pagination.empty(); + pagination.removeData("twbs-pagination"); + if (hits.length === 0) { + $('#search-results>.sr-items').html('

      No results found

      '); + } else { + pagination.twbsPagination({ + first: pagination.data('first'), + prev: pagination.data('prev'), + next: pagination.data('next'), + last: pagination.data('last'), + totalPages: Math.ceil(hits.length / numPerPage), + visiblePages: 5, + onPageClick: function (event, page) { + var start = (page - 1) * numPerPage; + var curHits = hits.slice(start, start + numPerPage); + $('#search-results>.sr-items').empty().append( + curHits.map(function (hit) { + var currentUrl = window.location.href; + var itemRawHref = relativeUrlToAbsoluteUrl(currentUrl, relHref + hit.href); + var itemTitle = hit.title; + var itemBrief = extractContentBrief(hit.keywords); + + var itemNode = $('
      ').attr('class', 'sr-item'); + var itemTitleNode = $('
      ').attr('class', 'item-title').append($('').attr('href', 'itemHref').attr("target", "_blank").attr("rel", "noopener noreferrer").text(itemTitle)); + var itemHrefNode = $('
      ').attr('class', 'item-href').text(itemRawHref); + var itemBriefNode = $('
      ').attr('class', 'item-brief').text(itemBrief); + itemNode.append(itemTitleNode).append(itemHrefNode).append(itemBriefNode); + return itemNode; + }) + ); + query.split(/\s+/).forEach(function (word) { + if (word !== '') { + $('#search-results>.sr-items *').mark(word); + } + }); + } + }); + } + } + }; + + // Update href in navbar + function renderNavbar() { + var navbar = $('#navbar ul')[0]; + if (typeof (navbar) === 'undefined') { + loadNavbar(); + } else { + $('#navbar ul a.active').parents('li').addClass(active); + renderBreadcrumb(); + showSearch(); + } + + function showSearch() { + if ($('#search-results').length !== 0) { + $('#search').show(); + $('body').trigger("searchEvent"); + } + } + + function loadNavbar() { + var navbarPath = $("meta[property='docfx\\:navrel']").attr("content"); + if (!navbarPath) { + return; + } + navbarPath = navbarPath.replace(/\\/g, '/'); + var tocPath = $("meta[property='docfx\\:tocrel']").attr("content") || ''; + if (tocPath) tocPath = tocPath.replace(/\\/g, '/'); + $.get(navbarPath, function (data) { + $(data).find("#toc>ul").appendTo("#navbar"); + showSearch(); + var index = navbarPath.lastIndexOf('/'); + var navrel = ''; + if (index > -1) { + navrel = navbarPath.substr(0, index + 1); + } + $('#navbar>ul').addClass('navbar-nav'); + var currentAbsPath = util.getCurrentWindowAbsolutePath(); + // set active item + $('#navbar').find('a[href]').each(function (i, e) { + var href = $(e).attr("href"); + if (util.isRelativePath(href)) { + href = navrel + href; + $(e).attr("href", href); + + var isActive = false; + var originalHref = e.name; + if (originalHref) { + originalHref = navrel + originalHref; + if (util.getDirectory(util.getAbsolutePath(originalHref)) === util.getDirectory(util.getAbsolutePath(tocPath))) { + isActive = true; + } + } else { + if (util.getAbsolutePath(href) === currentAbsPath) { + var dropdown = $(e).attr('data-toggle') == "dropdown" + if (!dropdown) { + isActive = true; + } + } + } + if (isActive) { + $(e).addClass(active); + } + } + }); + renderNavbar(); + }); + } + } + + function renderSidebar() { + var sidetoc = $('#sidetoggle .sidetoc')[0]; + if (typeof (sidetoc) === 'undefined') { + loadToc(); + } else { + registerTocEvents(); + if ($('footer').is(':visible')) { + $('.sidetoc').addClass('shiftup'); + } + + // Scroll to active item + var top = 0; + $('#toc a.active').parents('li').each(function (i, e) { + $(e).addClass(active).addClass(expanded); + $(e).children('a').addClass(active); + }) + $('#toc a.active').parents('li').each(function (i, e) { + top += $(e).position().top; + }) + $('.sidetoc').scrollTop(top - 50); + + if ($('footer').is(':visible')) { + $('.sidetoc').addClass('shiftup'); + } + + renderBreadcrumb(); + } + + function registerTocEvents() { + var tocFilterInput = $('#toc_filter_input'); + var tocFilterClearButton = $('#toc_filter_clear'); + + $('.toc .nav > li > .expand-stub').click(function (e) { + $(e.target).parent().toggleClass(expanded); + }); + $('.toc .nav > li > .expand-stub + a:not([href])').click(function (e) { + $(e.target).parent().toggleClass(expanded); + }); + tocFilterInput.on('input', function (e) { + var val = this.value; + //Save filter string to local session storage + if (typeof(Storage) !== "undefined") { + try { + sessionStorage.filterString = val; + } + catch(e) + {} + } + if (val === '') { + // Clear 'filtered' class + $('#toc li').removeClass(filtered).removeClass(hide); + tocFilterClearButton.fadeOut(); + return; + } + tocFilterClearButton.fadeIn(); + + // set all parent nodes status + $('#toc li>a').filter(function (i, e) { + return $(e).siblings().length > 0 + }).each(function (i, anchor) { + var parent = $(anchor).parent(); + parent.addClass(hide); + parent.removeClass(show); + parent.removeClass(filtered); + }) + + // Get leaf nodes + $('#toc li>a').filter(function (i, e) { + return $(e).siblings().length === 0 + }).each(function (i, anchor) { + var text = $(anchor).attr('title'); + var parent = $(anchor).parent(); + var parentNodes = parent.parents('ul>li'); + for (var i = 0; i < parentNodes.length; i++) { + var parentText = $(parentNodes[i]).children('a').attr('title'); + if (parentText) text = parentText + '.' + text; + }; + if (filterNavItem(text, val)) { + parent.addClass(show); + parent.removeClass(hide); + } else { + parent.addClass(hide); + parent.removeClass(show); + } + }); + $('#toc li>a').filter(function (i, e) { + return $(e).siblings().length > 0 + }).each(function (i, anchor) { + var parent = $(anchor).parent(); + if (parent.find('li.show').length > 0) { + parent.addClass(show); + parent.addClass(filtered); + parent.removeClass(hide); + } else { + parent.addClass(hide); + parent.removeClass(show); + parent.removeClass(filtered); + } + }) + + function filterNavItem(name, text) { + if (!text) return true; + if (name && name.toLowerCase().indexOf(text.toLowerCase()) > -1) return true; + return false; + } + }); + + // toc filter clear button + tocFilterClearButton.hide(); + tocFilterClearButton.on("click", function(e){ + tocFilterInput.val(""); + tocFilterInput.trigger('input'); + if (typeof(Storage) !== "undefined") { + try { + sessionStorage.filterString = ""; + } + catch(e) + {} + } + }); + + //Set toc filter from local session storage on page load + if (typeof(Storage) !== "undefined") { + try { + tocFilterInput.val(sessionStorage.filterString); + tocFilterInput.trigger('input'); + } + catch(e) + {} + } + } + + function loadToc() { + var tocPath = $("meta[property='docfx\\:tocrel']").attr("content"); + if (!tocPath) { + return; + } + tocPath = tocPath.replace(/\\/g, '/'); + $('#sidetoc').load(tocPath + " #sidetoggle > div", function () { + var index = tocPath.lastIndexOf('/'); + var tocrel = ''; + if (index > -1) { + tocrel = tocPath.substr(0, index + 1); + } + var currentHref = util.getCurrentWindowAbsolutePath(); + if(!currentHref.endsWith('.html')) { + currentHref += '.html'; + } + $('#sidetoc').find('a[href]').each(function (i, e) { + var href = $(e).attr("href"); + if (util.isRelativePath(href)) { + href = tocrel + href; + $(e).attr("href", href); + } + + if (util.getAbsolutePath(e.href) === currentHref) { + $(e).addClass(active); + } + + $(e).breakWord(); + }); + + renderSidebar(); + }); + } + } + + function renderBreadcrumb() { + var breadcrumb = []; + $('#navbar a.active').each(function (i, e) { + breadcrumb.push({ + href: e.href, + name: e.innerHTML + }); + }) + $('#toc a.active').each(function (i, e) { + breadcrumb.push({ + href: e.href, + name: e.innerHTML + }); + }) + + var html = util.formList(breadcrumb, 'breadcrumb'); + $('#breadcrumb').html(html); + } + + //Setup Affix + function renderAffix() { + var hierarchy = getHierarchy(); + if (!hierarchy || hierarchy.length <= 0) { + $("#affix").hide(); + } + else { + var html = util.formList(hierarchy, ['nav', 'bs-docs-sidenav']); + $("#affix>div").empty().append(html); + if ($('footer').is(':visible')) { + $(".sideaffix").css("bottom", "70px"); + } + $('#affix a').click(function(e) { + var scrollspy = $('[data-spy="scroll"]').data()['bs.scrollspy']; + var target = e.target.hash; + if (scrollspy && target) { + scrollspy.activate(target); + } + }); + } + + function getHierarchy() { + // supported headers are h1, h2, h3, and h4 + var $headers = $($.map(['h1', 'h2', 'h3', 'h4'], function (h) { return ".article article " + h; }).join(", ")); + + // a stack of hierarchy items that are currently being built + var stack = []; + $headers.each(function (i, e) { + if (!e.id) { + return; + } + + var item = { + name: htmlEncode($(e).text()), + href: "#" + e.id, + items: [] + }; + + if (!stack.length) { + stack.push({ type: e.tagName, siblings: [item] }); + return; + } + + var frame = stack[stack.length - 1]; + if (e.tagName === frame.type) { + frame.siblings.push(item); + } else if (e.tagName[1] > frame.type[1]) { + // we are looking at a child of the last element of frame.siblings. + // push a frame onto the stack. After we've finished building this item's children, + // we'll attach it as a child of the last element + stack.push({ type: e.tagName, siblings: [item] }); + } else { // e.tagName[1] < frame.type[1] + // we are looking at a sibling of an ancestor of the current item. + // pop frames from the stack, building items as we go, until we reach the correct level at which to attach this item. + while (e.tagName[1] < stack[stack.length - 1].type[1]) { + buildParent(); + } + if (e.tagName === stack[stack.length - 1].type) { + stack[stack.length - 1].siblings.push(item); + } else { + stack.push({ type: e.tagName, siblings: [item] }); + } + } + }); + while (stack.length > 1) { + buildParent(); + } + + function buildParent() { + var childrenToAttach = stack.pop(); + var parentFrame = stack[stack.length - 1]; + var parent = parentFrame.siblings[parentFrame.siblings.length - 1]; + $.each(childrenToAttach.siblings, function (i, child) { + parent.items.push(child); + }); + } + if (stack.length > 0) { + + var topLevel = stack.pop().siblings; + if (topLevel.length === 1) { // if there's only one topmost header, dump it + return topLevel[0].items; + } + return topLevel; + } + return undefined; + } + + function htmlEncode(str) { + if (!str) return str; + return str + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, ''') + .replace(//g, '>'); + } + } + + // Show footer + function renderFooter() { + initFooter(); + $(window).on("scroll", showFooterCore); + + function initFooter() { + if (needFooter()) { + shiftUpBottomCss(); + $("footer").show(); + } else { + resetBottomCss(); + $("footer").hide(); + } + } + + function showFooterCore() { + if (needFooter()) { + shiftUpBottomCss(); + $("footer").fadeIn(); + } else { + resetBottomCss(); + $("footer").fadeOut(); + } + } + + function needFooter() { + var scrollHeight = $(document).height(); + var scrollPosition = $(window).height() + $(window).scrollTop(); + return (scrollHeight - scrollPosition) < 1; + } + + function resetBottomCss() { + $(".sidetoc").removeClass("shiftup"); + $(".sideaffix").removeClass("shiftup"); + } + + function shiftUpBottomCss() { + $(".sidetoc").addClass("shiftup"); + $(".sideaffix").addClass("shiftup"); + } + } + + function renderLogo() { + // For LOGO SVG + // Replace SVG with inline SVG + // http://stackoverflow.com/questions/11978995/how-to-change-color-of-svg-image-using-css-jquery-svg-image-replacement + jQuery('img.svg').each(function () { + var $img = jQuery(this); + var imgID = $img.attr('id'); + var imgClass = $img.attr('class'); + var imgURL = $img.attr('src'); + + jQuery.get(imgURL, function (data) { + // Get the SVG tag, ignore the rest + var $svg = jQuery(data).find('svg'); + + // Add replaced image's ID to the new SVG + if (typeof imgID !== 'undefined') { + $svg = $svg.attr('id', imgID); + } + // Add replaced image's classes to the new SVG + if (typeof imgClass !== 'undefined') { + $svg = $svg.attr('class', imgClass + ' replaced-svg'); + } + + // Remove any invalid XML tags as per http://validator.w3.org + $svg = $svg.removeAttr('xmlns:a'); + + // Replace image with new SVG + $img.replaceWith($svg); + + }, 'xml'); + }); + } + + function renderTabs() { + var contentAttrs = { + id: 'data-bi-id', + name: 'data-bi-name', + type: 'data-bi-type' + }; + + var Tab = (function () { + function Tab(li, a, section) { + this.li = li; + this.a = a; + this.section = section; + } + Object.defineProperty(Tab.prototype, "tabIds", { + get: function () { return this.a.getAttribute('data-tab').split(' '); }, + enumerable: true, + configurable: true + }); + Object.defineProperty(Tab.prototype, "condition", { + get: function () { return this.a.getAttribute('data-condition'); }, + enumerable: true, + configurable: true + }); + Object.defineProperty(Tab.prototype, "visible", { + get: function () { return !this.li.hasAttribute('hidden'); }, + set: function (value) { + if (value) { + this.li.removeAttribute('hidden'); + this.li.removeAttribute('aria-hidden'); + } + else { + this.li.setAttribute('hidden', 'hidden'); + this.li.setAttribute('aria-hidden', 'true'); + } + }, + enumerable: true, + configurable: true + }); + Object.defineProperty(Tab.prototype, "selected", { + get: function () { return !this.section.hasAttribute('hidden'); }, + set: function (value) { + if (value) { + this.a.setAttribute('aria-selected', 'true'); + this.a.tabIndex = 0; + this.section.removeAttribute('hidden'); + this.section.removeAttribute('aria-hidden'); + } + else { + this.a.setAttribute('aria-selected', 'false'); + this.a.tabIndex = -1; + this.section.setAttribute('hidden', 'hidden'); + this.section.setAttribute('aria-hidden', 'true'); + } + }, + enumerable: true, + configurable: true + }); + Tab.prototype.focus = function () { + this.a.focus(); + }; + return Tab; + }()); + + initTabs(document.body); + + function initTabs(container) { + var queryStringTabs = readTabsQueryStringParam(); + var elements = container.querySelectorAll('.tabGroup'); + var state = { groups: [], selectedTabs: [] }; + for (var i = 0; i < elements.length; i++) { + var group = initTabGroup(elements.item(i)); + if (!group.independent) { + updateVisibilityAndSelection(group, state); + state.groups.push(group); + } + } + container.addEventListener('click', function (event) { return handleClick(event, state); }); + if (state.groups.length === 0) { + return state; + } + selectTabs(queryStringTabs); + updateTabsQueryStringParam(state); + notifyContentUpdated(); + return state; + } + + function initTabGroup(element) { + var group = { + independent: element.hasAttribute('data-tab-group-independent'), + tabs: [] + }; + var li = element.firstElementChild.firstElementChild; + while (li) { + var a = li.firstElementChild; + a.setAttribute(contentAttrs.name, 'tab'); + var dataTab = a.getAttribute('data-tab').replace(/\+/g, ' '); + a.setAttribute('data-tab', dataTab); + var section = element.querySelector("[id=\"" + a.getAttribute('aria-controls') + "\"]"); + var tab = new Tab(li, a, section); + group.tabs.push(tab); + li = li.nextElementSibling; + } + element.setAttribute(contentAttrs.name, 'tab-group'); + element.tabGroup = group; + return group; + } + + function updateVisibilityAndSelection(group, state) { + var anySelected = false; + var firstVisibleTab; + for (var _i = 0, _a = group.tabs; _i < _a.length; _i++) { + var tab = _a[_i]; + tab.visible = tab.condition === null || state.selectedTabs.indexOf(tab.condition) !== -1; + if (tab.visible) { + if (!firstVisibleTab) { + firstVisibleTab = tab; + } + } + tab.selected = tab.visible && arraysIntersect(state.selectedTabs, tab.tabIds); + anySelected = anySelected || tab.selected; + } + if (!anySelected) { + for (var _b = 0, _c = group.tabs; _b < _c.length; _b++) { + var tabIds = _c[_b].tabIds; + for (var _d = 0, tabIds_1 = tabIds; _d < tabIds_1.length; _d++) { + var tabId = tabIds_1[_d]; + var index = state.selectedTabs.indexOf(tabId); + if (index === -1) { + continue; + } + state.selectedTabs.splice(index, 1); + } + } + var tab = firstVisibleTab; + tab.selected = true; + state.selectedTabs.push(tab.tabIds[0]); + } + } + + function getTabInfoFromEvent(event) { + if (!(event.target instanceof HTMLElement)) { + return null; + } + var anchor = event.target.closest('a[data-tab]'); + if (anchor === null) { + return null; + } + var tabIds = anchor.getAttribute('data-tab').split(' '); + var group = anchor.parentElement.parentElement.parentElement.tabGroup; + if (group === undefined) { + return null; + } + return { tabIds: tabIds, group: group, anchor: anchor }; + } + + function handleClick(event, state) { + var info = getTabInfoFromEvent(event); + if (info === null) { + return; + } + event.preventDefault(); + info.anchor.href = 'javascript:'; + setTimeout(function () { return info.anchor.href = $('#' + info.anchor.getAttribute('aria-controls')); }); + var tabIds = info.tabIds, group = info.group; + var originalTop = info.anchor.getBoundingClientRect().top; + if (group.independent) { + for (var _i = 0, _a = group.tabs; _i < _a.length; _i++) { + var tab = _a[_i]; + tab.selected = arraysIntersect(tab.tabIds, tabIds); + } + } + else { + if (arraysIntersect(state.selectedTabs, tabIds)) { + return; + } + var previousTabId = group.tabs.filter(function (t) { return t.selected; })[0].tabIds[0]; + state.selectedTabs.splice(state.selectedTabs.indexOf(previousTabId), 1, tabIds[0]); + for (var _b = 0, _c = state.groups; _b < _c.length; _b++) { + var group_1 = _c[_b]; + updateVisibilityAndSelection(group_1, state); + } + updateTabsQueryStringParam(state); + } + notifyContentUpdated(); + var top = info.anchor.getBoundingClientRect().top; + if (top !== originalTop && event instanceof MouseEvent) { + window.scrollTo(0, window.pageYOffset + top - originalTop); + } + } + + function selectTabs(tabIds) { + for (var _i = 0, tabIds_1 = tabIds; _i < tabIds_1.length; _i++) { + var tabId = tabIds_1[_i]; + var a = document.querySelector(".tabGroup > ul > li > a[data-tab=\"" + tabId + "\"]:not([hidden])"); + if (a === null) { + return; + } + a.dispatchEvent(new CustomEvent('click', { bubbles: true })); + } + } + + function readTabsQueryStringParam() { + var qs = parseQueryString(window.location.search); + var t = qs.tabs; + if (t === undefined || t === '') { + return []; + } + return t.split(','); + } + + function updateTabsQueryStringParam(state) { + var qs = parseQueryString(window.location.search); + qs.tabs = state.selectedTabs.join(); + var url = location.protocol + "//" + location.host + location.pathname + "?" + toQueryString(qs) + location.hash; + if (location.href === url) { + return; + } + history.replaceState({}, document.title, url); + } + + function toQueryString(args) { + var parts = []; + for (var name_1 in args) { + if (args.hasOwnProperty(name_1) && args[name_1] !== '' && args[name_1] !== null && args[name_1] !== undefined) { + parts.push(encodeURIComponent(name_1) + '=' + encodeURIComponent(args[name_1])); + } + } + return parts.join('&'); + } + + function parseQueryString(queryString) { + var match; + var pl = /\+/g; + var search = /([^&=]+)=?([^&]*)/g; + var decode = function (s) { return decodeURIComponent(s.replace(pl, ' ')); }; + if (queryString === undefined) { + queryString = ''; + } + queryString = queryString.substring(1); + var urlParams = {}; + while (match = search.exec(queryString)) { + urlParams[decode(match[1])] = decode(match[2]); + } + return urlParams; + } + + function arraysIntersect(a, b) { + for (var _i = 0, a_1 = a; _i < a_1.length; _i++) { + var itemA = a_1[_i]; + for (var _a = 0, b_1 = b; _a < b_1.length; _a++) { + var itemB = b_1[_a]; + if (itemA === itemB) { + return true; + } + } + } + return false; + } + + function notifyContentUpdated() { + // Dispatch this event when needed + // window.dispatchEvent(new CustomEvent('content-update')); + } + } + + function utility() { + this.getAbsolutePath = getAbsolutePath; + this.isRelativePath = isRelativePath; + this.isAbsolutePath = isAbsolutePath; + this.getCurrentWindowAbsolutePath = getCurrentWindowAbsolutePath; + this.getDirectory = getDirectory; + this.formList = formList; + + function getAbsolutePath(href) { + if (isAbsolutePath(href)) return href; + var currentAbsPath = getCurrentWindowAbsolutePath(); + var stack = currentAbsPath.split("/"); + stack.pop(); + var parts = href.split("/"); + for (var i=0; i< parts.length; i++) { + if (parts[i] == ".") continue; + if (parts[i] == ".." && stack.length > 0) + stack.pop(); + else + stack.push(parts[i]); + } + var p = stack.join("/"); + return p; + } + + function isRelativePath(href) { + if (href === undefined || href === '' || href[0] === '/') { + return false; + } + return !isAbsolutePath(href); + } + + function isAbsolutePath(href) { + return (/^(?:[a-z]+:)?\/\//i).test(href); + } + + function getCurrentWindowAbsolutePath() { + return window.location.origin + window.location.pathname; + } + function getDirectory(href) { + if (!href) return ''; + var index = href.lastIndexOf('/'); + if (index == -1) return ''; + if (index > -1) { + return href.substr(0, index); + } + } + + function formList(item, classes) { + var level = 1; + var model = { + items: item + }; + var cls = [].concat(classes).join(" "); + return getList(model, cls); + + function getList(model, cls) { + if (!model || !model.items) return null; + var l = model.items.length; + if (l === 0) return null; + var html = ''; + return html; + } + } + + /** + * Add into long word. + * @param {String} text - The word to break. It should be in plain text without HTML tags. + */ + function breakPlainText(text) { + if (!text) return text; + return text.replace(/([a-z])([A-Z])|(\.)(\w)/g, '$1$3$2$4') + } + + /** + * Add into long word. The jQuery element should contain no html tags. + * If the jQuery element contains tags, this function will not change the element. + */ + $.fn.breakWord = function () { + if (this.html() == this.text()) { + this.html(function (index, text) { + return breakPlainText(text); + }) + } + return this; + } + } + + // adjusted from https://stackoverflow.com/a/13067009/1523776 + function workAroundFixedHeaderForAnchors() { + var HISTORY_SUPPORT = !!(history && history.pushState); + var ANCHOR_REGEX = /^#[^ ]+$/; + + function getFixedOffset() { + return $('header').first().height(); + } + + /** + * If the provided href is an anchor which resolves to an element on the + * page, scroll to it. + * @param {String} href + * @return {Boolean} - Was the href an anchor. + */ + function scrollIfAnchor(href, pushToHistory) { + var match, rect, anchorOffset; + + if (!ANCHOR_REGEX.test(href)) { + return false; + } + + match = document.getElementById(href.slice(1)); + + if (match) { + rect = match.getBoundingClientRect(); + anchorOffset = window.pageYOffset + rect.top - getFixedOffset(); + window.scrollTo(window.pageXOffset, anchorOffset); + + // Add the state to history as-per normal anchor links + if (HISTORY_SUPPORT && pushToHistory) { + history.pushState({}, document.title, location.pathname + href); + } + } + + return !!match; + } + + /** + * Attempt to scroll to the current location's hash. + */ + function scrollToCurrent() { + scrollIfAnchor(window.location.hash); + } + + /** + * If the click event's target was an anchor, fix the scroll position. + */ + function delegateAnchors(e) { + var elem = e.target; + + if (scrollIfAnchor(elem.getAttribute('href'), true)) { + e.preventDefault(); + } + } + + $(window).on('hashchange', scrollToCurrent); + + $(window).on('load', function () { + // scroll to the anchor if present, offset by the header + scrollToCurrent(); + }); + + $(document).ready(function () { + // Exclude tabbed content case + $('a:not([data-tab])').click(function (e) { delegateAnchors(e); }); + }); + } +}); diff --git a/csharp/ApiDocs/_exported_templates/default/styles/docfx.vendor.css b/csharp/ApiDocs/_exported_templates/default/styles/docfx.vendor.css new file mode 100644 index 0000000000..609602eb13 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/docfx.vendor.css @@ -0,0 +1,1513 @@ +/*! + * Bootstrap v3.4.1 (https://getbootstrap.com/) + * Copyright 2011-2019 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */ +/*! normalize.css v3.0.3 | MIT License | github.com/necolas/normalize.css */ +html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%} +article,aside,details,figcaption,figure,footer,header,hgroup,main,menu,nav,section,summary{display:block} +audio,canvas,progress,video{display:inline-block;vertical-align:baseline} +audio:not([controls]){display:none;height:0} +[hidden],template{display:none} +a:active,a:hover{outline:0} +abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;-moz-text-decoration:underline dotted;text-decoration:underline dotted} +b,optgroup,strong{font-weight:700} +dfn{font-style:italic} +h1{margin:.67em 0} +mark{background:#ff0;color:#000} +sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline} +sup{top:-.5em} +sub{bottom:-.25em} +img{border:0;vertical-align:middle} +svg:not(:root){overflow:hidden} +hr{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;height:0} +code,kbd,pre,samp{font-size:1em} +button,input,optgroup,select,textarea{color:inherit;font:inherit;margin:0} +button{overflow:visible} +button,select{text-transform:none} +button,html input[type=button],input[type=reset],input[type=submit]{-webkit-appearance:button;cursor:pointer} +button[disabled],html input[disabled]{cursor:default} +button::-moz-focus-inner,input::-moz-focus-inner{border:0;padding:0} +input{line-height:normal} +input[type=checkbox],input[type=radio]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:0} +input[type=number]::-webkit-inner-spin-button,input[type=number]::-webkit-outer-spin-button{height:auto} +input[type=search]{-webkit-appearance:textfield;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box} +input[type=search]::-webkit-search-cancel-button,input[type=search]::-webkit-search-decoration{-webkit-appearance:none} +textarea{overflow:auto} +td,th{padding:0} +/*! Source: https://github.com/h5bp/html5-boilerplate/blob/master/src/css/main.css */ +@media print{ +*,:after,:before{color:#000!important;text-shadow:none!important;background:0 0!important;-webkit-box-shadow:none!important;box-shadow:none!important} +a,a:visited{text-decoration:underline} +a[href]:after{content:" (" attr(href) ")"} +abbr[title]:after{content:" (" attr(title) ")"} +a[href^="#"]:after,a[href^="javascript:"]:after{content:""} +blockquote,pre{border:1px solid #999;page-break-inside:avoid} +thead{display:table-header-group} +img,tr{page-break-inside:avoid} +img{max-width:100%!important} +h2,h3,p{orphans:3;widows:3} +h2,h3{page-break-after:avoid} +.navbar{display:none} +.btn>.caret,.dropup>.btn>.caret{border-top-color:#000!important} +.label{border:1px solid #000} +.table{border-collapse:collapse!important} +.table td,.table th{background-color:#fff!important} +.table-bordered td,.table-bordered th{border:1px solid #ddd!important} +} +@font-face{font-family:"Glyphicons Halflings";src:url("../fonts/glyphicons-halflings-regular.eot");src:url("../fonts/glyphicons-halflings-regular.eot?#iefix") format("embedded-opentype"),url("../fonts/glyphicons-halflings-regular.woff2") format("woff2"),url("../fonts/glyphicons-halflings-regular.woff") format("woff"),url("../fonts/glyphicons-halflings-regular.ttf") format("truetype"),url("../fonts/glyphicons-halflings-regular.svg#glyphicons_halflingsregular") format("svg")} +.glyphicon{position:relative;top:1px;display:inline-block;font-family:"Glyphicons Halflings";font-style:normal;font-weight:400;line-height:1;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale} +.glyphicon-asterisk:before{content:"\002a"} +.glyphicon-plus:before{content:"\002b"} +.glyphicon-eur:before,.glyphicon-euro:before{content:"\20ac"} +.glyphicon-minus:before{content:"\2212"} +.glyphicon-cloud:before{content:"\2601"} +.glyphicon-envelope:before{content:"\2709"} +.glyphicon-pencil:before{content:"\270f"} +.glyphicon-glass:before{content:"\e001"} +.glyphicon-music:before{content:"\e002"} +.glyphicon-search:before{content:"\e003"} +.glyphicon-heart:before{content:"\e005"} +.glyphicon-star:before{content:"\e006"} +.glyphicon-star-empty:before{content:"\e007"} +.glyphicon-user:before{content:"\e008"} +.glyphicon-film:before{content:"\e009"} +.glyphicon-th-large:before{content:"\e010"} +.glyphicon-th:before{content:"\e011"} +.glyphicon-th-list:before{content:"\e012"} +.glyphicon-ok:before{content:"\e013"} +.glyphicon-remove:before{content:"\e014"} +.glyphicon-zoom-in:before{content:"\e015"} +.glyphicon-zoom-out:before{content:"\e016"} +.glyphicon-off:before{content:"\e017"} +.glyphicon-signal:before{content:"\e018"} +.glyphicon-cog:before{content:"\e019"} +.glyphicon-trash:before{content:"\e020"} +.glyphicon-home:before{content:"\e021"} +.glyphicon-file:before{content:"\e022"} +.glyphicon-time:before{content:"\e023"} +.glyphicon-road:before{content:"\e024"} +.glyphicon-download-alt:before{content:"\e025"} +.glyphicon-download:before{content:"\e026"} +.glyphicon-upload:before{content:"\e027"} +.glyphicon-inbox:before{content:"\e028"} +.glyphicon-play-circle:before{content:"\e029"} +.glyphicon-repeat:before{content:"\e030"} +.glyphicon-refresh:before{content:"\e031"} +.glyphicon-list-alt:before{content:"\e032"} +.glyphicon-lock:before{content:"\e033"} +.glyphicon-flag:before{content:"\e034"} +.glyphicon-headphones:before{content:"\e035"} +.glyphicon-volume-off:before{content:"\e036"} +.glyphicon-volume-down:before{content:"\e037"} +.glyphicon-volume-up:before{content:"\e038"} +.glyphicon-qrcode:before{content:"\e039"} +.glyphicon-barcode:before{content:"\e040"} +.glyphicon-tag:before{content:"\e041"} +.glyphicon-tags:before{content:"\e042"} +.glyphicon-book:before{content:"\e043"} +.glyphicon-bookmark:before{content:"\e044"} +.glyphicon-print:before{content:"\e045"} +.glyphicon-camera:before{content:"\e046"} +.glyphicon-font:before{content:"\e047"} +.glyphicon-bold:before{content:"\e048"} +.glyphicon-italic:before{content:"\e049"} +.glyphicon-text-height:before{content:"\e050"} +.glyphicon-text-width:before{content:"\e051"} +.glyphicon-align-left:before{content:"\e052"} +.glyphicon-align-center:before{content:"\e053"} +.glyphicon-align-right:before{content:"\e054"} +.glyphicon-align-justify:before{content:"\e055"} +.glyphicon-list:before{content:"\e056"} +.glyphicon-indent-left:before{content:"\e057"} +.glyphicon-indent-right:before{content:"\e058"} +.glyphicon-facetime-video:before{content:"\e059"} +.glyphicon-picture:before{content:"\e060"} +.glyphicon-map-marker:before{content:"\e062"} +.glyphicon-adjust:before{content:"\e063"} +.glyphicon-tint:before{content:"\e064"} +.glyphicon-edit:before{content:"\e065"} +.glyphicon-share:before{content:"\e066"} +.glyphicon-check:before{content:"\e067"} +.glyphicon-move:before{content:"\e068"} +.glyphicon-step-backward:before{content:"\e069"} +.glyphicon-fast-backward:before{content:"\e070"} +.glyphicon-backward:before{content:"\e071"} +.glyphicon-play:before{content:"\e072"} +.glyphicon-pause:before{content:"\e073"} +.glyphicon-stop:before{content:"\e074"} +.glyphicon-forward:before{content:"\e075"} +.glyphicon-fast-forward:before{content:"\e076"} +.glyphicon-step-forward:before{content:"\e077"} +.glyphicon-eject:before{content:"\e078"} +.glyphicon-chevron-left:before{content:"\e079"} +.glyphicon-chevron-right:before{content:"\e080"} +.glyphicon-plus-sign:before{content:"\e081"} +.glyphicon-minus-sign:before{content:"\e082"} +.glyphicon-remove-sign:before{content:"\e083"} +.glyphicon-ok-sign:before{content:"\e084"} +.glyphicon-question-sign:before{content:"\e085"} +.glyphicon-info-sign:before{content:"\e086"} +.glyphicon-screenshot:before{content:"\e087"} +.glyphicon-remove-circle:before{content:"\e088"} +.glyphicon-ok-circle:before{content:"\e089"} +.glyphicon-ban-circle:before{content:"\e090"} +.glyphicon-arrow-left:before{content:"\e091"} +.glyphicon-arrow-right:before{content:"\e092"} +.glyphicon-arrow-up:before{content:"\e093"} +.glyphicon-arrow-down:before{content:"\e094"} +.glyphicon-share-alt:before{content:"\e095"} +.glyphicon-resize-full:before{content:"\e096"} +.glyphicon-resize-small:before{content:"\e097"} +.glyphicon-exclamation-sign:before{content:"\e101"} +.glyphicon-gift:before{content:"\e102"} +.glyphicon-leaf:before{content:"\e103"} +.glyphicon-fire:before{content:"\e104"} +.glyphicon-eye-open:before{content:"\e105"} +.glyphicon-eye-close:before{content:"\e106"} +.glyphicon-warning-sign:before{content:"\e107"} +.glyphicon-plane:before{content:"\e108"} +.glyphicon-calendar:before{content:"\e109"} +.glyphicon-random:before{content:"\e110"} +.glyphicon-comment:before{content:"\e111"} +.glyphicon-magnet:before{content:"\e112"} +.glyphicon-chevron-up:before{content:"\e113"} +.glyphicon-chevron-down:before{content:"\e114"} +.glyphicon-retweet:before{content:"\e115"} +.glyphicon-shopping-cart:before{content:"\e116"} +.glyphicon-folder-close:before{content:"\e117"} +.glyphicon-folder-open:before{content:"\e118"} +.glyphicon-resize-vertical:before{content:"\e119"} +.glyphicon-resize-horizontal:before{content:"\e120"} +.glyphicon-hdd:before{content:"\e121"} +.glyphicon-bullhorn:before{content:"\e122"} +.glyphicon-bell:before{content:"\e123"} +.glyphicon-certificate:before{content:"\e124"} +.glyphicon-thumbs-up:before{content:"\e125"} +.glyphicon-thumbs-down:before{content:"\e126"} +.glyphicon-hand-right:before{content:"\e127"} +.glyphicon-hand-left:before{content:"\e128"} +.glyphicon-hand-up:before{content:"\e129"} +.glyphicon-hand-down:before{content:"\e130"} +.glyphicon-circle-arrow-right:before{content:"\e131"} +.glyphicon-circle-arrow-left:before{content:"\e132"} +.glyphicon-circle-arrow-up:before{content:"\e133"} +.glyphicon-circle-arrow-down:before{content:"\e134"} +.glyphicon-globe:before{content:"\e135"} +.glyphicon-wrench:before{content:"\e136"} +.glyphicon-tasks:before{content:"\e137"} +.glyphicon-filter:before{content:"\e138"} +.glyphicon-briefcase:before{content:"\e139"} +.glyphicon-fullscreen:before{content:"\e140"} +.glyphicon-dashboard:before{content:"\e141"} +.glyphicon-paperclip:before{content:"\e142"} +.glyphicon-heart-empty:before{content:"\e143"} +.glyphicon-link:before{content:"\e144"} +.glyphicon-phone:before{content:"\e145"} +.glyphicon-pushpin:before{content:"\e146"} +.glyphicon-usd:before{content:"\e148"} +.glyphicon-gbp:before{content:"\e149"} +.glyphicon-sort:before{content:"\e150"} +.glyphicon-sort-by-alphabet:before{content:"\e151"} +.glyphicon-sort-by-alphabet-alt:before{content:"\e152"} +.glyphicon-sort-by-order:before{content:"\e153"} +.glyphicon-sort-by-order-alt:before{content:"\e154"} +.glyphicon-sort-by-attributes:before{content:"\e155"} +.glyphicon-sort-by-attributes-alt:before{content:"\e156"} +.glyphicon-unchecked:before{content:"\e157"} +.glyphicon-expand:before{content:"\e158"} +.glyphicon-collapse-down:before{content:"\e159"} +.glyphicon-collapse-up:before{content:"\e160"} +.glyphicon-log-in:before{content:"\e161"} +.glyphicon-flash:before{content:"\e162"} +.glyphicon-log-out:before{content:"\e163"} +.glyphicon-new-window:before{content:"\e164"} +.glyphicon-record:before{content:"\e165"} +.glyphicon-save:before{content:"\e166"} +.glyphicon-open:before{content:"\e167"} +.glyphicon-saved:before{content:"\e168"} +.glyphicon-import:before{content:"\e169"} +.glyphicon-export:before{content:"\e170"} +.glyphicon-send:before{content:"\e171"} +.glyphicon-floppy-disk:before{content:"\e172"} +.glyphicon-floppy-saved:before{content:"\e173"} +.glyphicon-floppy-remove:before{content:"\e174"} +.glyphicon-floppy-save:before{content:"\e175"} +.glyphicon-floppy-open:before{content:"\e176"} +.glyphicon-credit-card:before{content:"\e177"} +.glyphicon-transfer:before{content:"\e178"} +.glyphicon-cutlery:before{content:"\e179"} +.glyphicon-header:before{content:"\e180"} +.glyphicon-compressed:before{content:"\e181"} +.glyphicon-earphone:before{content:"\e182"} +.glyphicon-phone-alt:before{content:"\e183"} +.glyphicon-tower:before{content:"\e184"} +.glyphicon-stats:before{content:"\e185"} +.glyphicon-sd-video:before{content:"\e186"} +.glyphicon-hd-video:before{content:"\e187"} +.glyphicon-subtitles:before{content:"\e188"} +.glyphicon-sound-stereo:before{content:"\e189"} +.glyphicon-sound-dolby:before{content:"\e190"} +.glyphicon-sound-5-1:before{content:"\e191"} +.glyphicon-sound-6-1:before{content:"\e192"} +.glyphicon-sound-7-1:before{content:"\e193"} +.glyphicon-copyright-mark:before{content:"\e194"} +.glyphicon-registration-mark:before{content:"\e195"} +.glyphicon-cloud-download:before{content:"\e197"} +.glyphicon-cloud-upload:before{content:"\e198"} +.glyphicon-tree-conifer:before{content:"\e199"} +.glyphicon-tree-deciduous:before{content:"\e200"} +.glyphicon-cd:before{content:"\e201"} +.glyphicon-save-file:before{content:"\e202"} +.glyphicon-open-file:before{content:"\e203"} +.glyphicon-level-up:before{content:"\e204"} +.glyphicon-copy:before{content:"\e205"} +.glyphicon-paste:before{content:"\e206"} +.glyphicon-alert:before{content:"\e209"} +.glyphicon-equalizer:before{content:"\e210"} +.glyphicon-king:before{content:"\e211"} +.glyphicon-queen:before{content:"\e212"} +.glyphicon-pawn:before{content:"\e213"} +.glyphicon-bishop:before{content:"\e214"} +.glyphicon-knight:before{content:"\e215"} +.glyphicon-baby-formula:before{content:"\e216"} +.glyphicon-tent:before{content:"\26fa"} +.glyphicon-blackboard:before{content:"\e218"} +.glyphicon-bed:before{content:"\e219"} +.glyphicon-apple:before{content:"\f8ff"} +.glyphicon-erase:before{content:"\e221"} +.glyphicon-hourglass:before{content:"\231b"} +.glyphicon-lamp:before{content:"\e223"} +.glyphicon-duplicate:before{content:"\e224"} +.glyphicon-piggy-bank:before{content:"\e225"} +.glyphicon-scissors:before{content:"\e226"} +.glyphicon-bitcoin:before,.glyphicon-btc:before,.glyphicon-xbt:before{content:"\e227"} +.glyphicon-jpy:before,.glyphicon-yen:before{content:"\00a5"} +.glyphicon-rub:before,.glyphicon-ruble:before{content:"\20bd"} +.glyphicon-scale:before{content:"\e230"} +.glyphicon-ice-lolly:before{content:"\e231"} +.glyphicon-ice-lolly-tasted:before{content:"\e232"} +.glyphicon-education:before{content:"\e233"} +.glyphicon-option-horizontal:before{content:"\e234"} +.glyphicon-option-vertical:before{content:"\e235"} +.glyphicon-menu-hamburger:before{content:"\e236"} +.glyphicon-modal-window:before{content:"\e237"} +.glyphicon-oil:before{content:"\e238"} +.glyphicon-grain:before{content:"\e239"} +.glyphicon-sunglasses:before{content:"\e240"} +.glyphicon-text-size:before{content:"\e241"} +.glyphicon-text-color:before{content:"\e242"} +.glyphicon-text-background:before{content:"\e243"} +.glyphicon-object-align-top:before{content:"\e244"} +.glyphicon-object-align-bottom:before{content:"\e245"} +.glyphicon-object-align-horizontal:before{content:"\e246"} +.glyphicon-object-align-left:before{content:"\e247"} +.glyphicon-object-align-vertical:before{content:"\e248"} +.glyphicon-object-align-right:before{content:"\e249"} +.glyphicon-triangle-right:before{content:"\e250"} +.glyphicon-triangle-left:before{content:"\e251"} +.glyphicon-triangle-bottom:before{content:"\e252"} +.glyphicon-triangle-top:before{content:"\e253"} +.glyphicon-console:before{content:"\e254"} +.glyphicon-superscript:before{content:"\e255"} +.glyphicon-subscript:before{content:"\e256"} +.glyphicon-menu-left:before{content:"\e257"} +.glyphicon-menu-right:before{content:"\e258"} +.glyphicon-menu-down:before{content:"\e259"} +.glyphicon-menu-up:before{content:"\e260"} +*,:after,:before{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box} +html{font-size:10px;-webkit-tap-highlight-color:transparent} +body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:1.42857143;color:#333;background-color:#fff} +button,input,select,textarea{font-family:inherit;font-size:inherit;line-height:inherit} +a{background-color:transparent;color:#337ab7;text-decoration:none} +a:focus,a:hover{color:#23527c;text-decoration:underline} +a:focus{outline:-webkit-focus-ring-color auto 5px;outline-offset:-2px} +figure{margin:0} +.carousel-inner>.item>a>img,.carousel-inner>.item>img,.img-responsive,.thumbnail a>img,.thumbnail>img{display:block;max-width:100%;height:auto} +.img-rounded{border-radius:6px} +.img-thumbnail{padding:4px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:.2s ease-in-out;-o-transition:.2s ease-in-out;transition:.2s ease-in-out;display:inline-block;max-width:100%;height:auto} +.img-circle{border-radius:50%} +hr{margin-top:20px;margin-bottom:20px;border:0;border-top:1px solid #eee} +.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);border:0} +.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} +[role=button]{cursor:pointer} +.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{font-family:inherit;font-weight:500;line-height:1.1;color:inherit} +.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-weight:400;line-height:1;color:#777} +.h1,.h2,.h3,h1,h2,h3{margin-top:20px;margin-bottom:10px} +.h1 .small,.h1 small,.h2 .small,.h2 small,.h3 .small,.h3 small,h1 .small,h1 small,h2 .small,h2 small,h3 .small,h3 small{font-size:65%} +.h4,.h5,.h6,h4,h5,h6{margin-top:10px;margin-bottom:10px} +.h4 .small,.h4 small,.h5 .small,.h5 small,.h6 .small,.h6 small,h4 .small,h4 small,h5 .small,h5 small,h6 .small,h6 small{font-size:75%} +.h1,h1{font-size:36px} +.h2,h2{font-size:30px} +.h3,h3{font-size:24px} +.h4,h4{font-size:18px} +.h5,h5{font-size:14px} +.h6,h6{font-size:12px} +p{margin:0 0 10px} +.lead{margin-bottom:20px;font-size:16px;font-weight:300;line-height:1.4} +@media (min-width:768px){ +.lead{font-size:21px} +.dl-horizontal dt{float:left;width:160px;clear:left;text-align:right;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.dl-horizontal dd{margin-left:180px} +} +.small,small{font-size:85%} +.mark,mark{padding:.2em;background-color:#fcf8e3} +.text-left{text-align:left} +.text-right{text-align:right} +.text-center{text-align:center} +.text-justify{text-align:justify} +.text-nowrap{white-space:nowrap} +.text-lowercase{text-transform:lowercase} +.text-uppercase{text-transform:uppercase} +.text-capitalize{text-transform:capitalize} +.text-muted{color:#777} +.text-primary{color:#337ab7} +a.text-primary:focus,a.text-primary:hover{color:#286090} +.text-success{color:#3c763d} +a.text-success:focus,a.text-success:hover{color:#2b542c} +.text-info{color:#31708f} +a.text-info:focus,a.text-info:hover{color:#245269} +.text-warning{color:#8a6d3b} +a.text-warning:focus,a.text-warning:hover{color:#66512c} +.text-danger{color:#a94442} +a.text-danger:focus,a.text-danger:hover{color:#843534} +.bg-primary{color:#fff;background-color:#337ab7} +a.bg-primary:focus,a.bg-primary:hover{background-color:#286090} +.bg-success{background-color:#dff0d8} +a.bg-success:focus,a.bg-success:hover{background-color:#c1e2b3} +.bg-info{background-color:#d9edf7} +a.bg-info:focus,a.bg-info:hover{background-color:#afd9ee} +.bg-warning{background-color:#fcf8e3} +a.bg-warning:focus,a.bg-warning:hover{background-color:#f7ecb5} +.bg-danger{background-color:#f2dede} +a.bg-danger:focus,a.bg-danger:hover{background-color:#e4b9b9} +.page-header{padding-bottom:9px;margin:40px 0 20px;border-bottom:1px solid #eee} +ol,ul{margin-top:0;margin-bottom:10px} +ol ol,ol ul,ul ol,ul ul{margin-bottom:0} +.list-unstyled{padding-left:0;list-style:none} +.list-inline{padding-left:0;list-style:none;margin-left:-5px} +.list-inline>li{display:inline-block;padding-right:5px;padding-left:5px} +dl{margin-top:0;margin-bottom:20px} +dd,dt{line-height:1.42857143} +dt{font-weight:700} +dd{margin-left:0} +abbr[data-original-title],abbr[title]{cursor:help} +.initialism{font-size:90%;text-transform:uppercase} +blockquote{padding:10px 20px;margin:0 0 20px;font-size:17.5px;border-left:5px solid #eee} +blockquote ol:last-child,blockquote p:last-child,blockquote ul:last-child{margin-bottom:0} +blockquote .small,blockquote footer,blockquote small{display:block;font-size:80%;line-height:1.42857143;color:#777} +blockquote .small:before,blockquote footer:before,blockquote small:before{content:"\2014 \00A0"} +.blockquote-reverse,blockquote.pull-right{padding-right:15px;padding-left:0;text-align:right;border-right:5px solid #eee;border-left:0} +.blockquote-reverse .small:before,.blockquote-reverse footer:before,.blockquote-reverse small:before,blockquote.pull-right .small:before,blockquote.pull-right footer:before,blockquote.pull-right small:before{content:""} +.blockquote-reverse .small:after,.blockquote-reverse footer:after,.blockquote-reverse small:after,blockquote.pull-right .small:after,blockquote.pull-right footer:after,blockquote.pull-right small:after{content:"\00A0 \2014"} +address{margin-bottom:20px;font-style:normal;line-height:1.42857143} +code,kbd,pre,samp{font-family:Menlo,Monaco,Consolas,"Courier New",monospace} +code{padding:2px 4px;font-size:90%;color:#c7254e;background-color:#f9f2f4;border-radius:4px} +kbd{padding:2px 4px;font-size:90%;color:#fff;background-color:#333;border-radius:3px;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.25);box-shadow:inset 0 -1px 0 rgba(0,0,0,.25)} +kbd kbd{padding:0;font-size:100%;font-weight:700;-webkit-box-shadow:none;box-shadow:none} +pre{overflow:auto;display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:1.42857143;color:#333;word-break:break-all;word-wrap:break-word;background-color:#f5f5f5;border:1px solid #ccc;border-radius:4px} +pre code{padding:0;font-size:inherit;color:inherit;white-space:pre-wrap;background-color:transparent;border-radius:0} +.pre-scrollable{max-height:340px;overflow-y:scroll} +.container{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto} +@media (min-width:768px){ +.container{width:750px} +} +@media (min-width:992px){ +.container{width:970px} +} +@media (min-width:1200px){ +.container{width:1170px} +} +.container-fluid{padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto} +.row{margin-right:-15px;margin-left:-15px} +.row-no-gutters{margin-right:0;margin-left:0} +.row-no-gutters [class*=col-]{padding-right:0;padding-left:0} +.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{position:relative;min-height:1px;padding-right:15px;padding-left:15px} +.col-xs-1,.col-xs-10,.col-xs-11,.col-xs-12,.col-xs-2,.col-xs-3,.col-xs-4,.col-xs-5,.col-xs-6,.col-xs-7,.col-xs-8,.col-xs-9{float:left} +.col-xs-12{width:100%} +.col-xs-11{width:91.66666667%} +.col-xs-10{width:83.33333333%} +.col-xs-9{width:75%} +.col-xs-8{width:66.66666667%} +.col-xs-7{width:58.33333333%} +.col-xs-6{width:50%} +.col-xs-5{width:41.66666667%} +.col-xs-4{width:33.33333333%} +.col-xs-3{width:25%} +.col-xs-2{width:16.66666667%} +.col-xs-1{width:8.33333333%} +.col-xs-pull-12{right:100%} +.col-xs-pull-11{right:91.66666667%} +.col-xs-pull-10{right:83.33333333%} +.col-xs-pull-9{right:75%} +.col-xs-pull-8{right:66.66666667%} +.col-xs-pull-7{right:58.33333333%} +.col-xs-pull-6{right:50%} +.col-xs-pull-5{right:41.66666667%} +.col-xs-pull-4{right:33.33333333%} +.col-xs-pull-3{right:25%} +.col-xs-pull-2{right:16.66666667%} +.col-xs-pull-1{right:8.33333333%} +.col-xs-pull-0{right:auto} +.col-xs-push-12{left:100%} +.col-xs-push-11{left:91.66666667%} +.col-xs-push-10{left:83.33333333%} +.col-xs-push-9{left:75%} +.col-xs-push-8{left:66.66666667%} +.col-xs-push-7{left:58.33333333%} +.col-xs-push-6{left:50%} +.col-xs-push-5{left:41.66666667%} +.col-xs-push-4{left:33.33333333%} +.col-xs-push-3{left:25%} +.col-xs-push-2{left:16.66666667%} +.col-xs-push-1{left:8.33333333%} +.col-xs-push-0{left:auto} +.col-xs-offset-12{margin-left:100%} +.col-xs-offset-11{margin-left:91.66666667%} +.col-xs-offset-10{margin-left:83.33333333%} +.col-xs-offset-9{margin-left:75%} +.col-xs-offset-8{margin-left:66.66666667%} +.col-xs-offset-7{margin-left:58.33333333%} +.col-xs-offset-6{margin-left:50%} +.col-xs-offset-5{margin-left:41.66666667%} +.col-xs-offset-4{margin-left:33.33333333%} +.col-xs-offset-3{margin-left:25%} +.col-xs-offset-2{margin-left:16.66666667%} +.col-xs-offset-1{margin-left:8.33333333%} +.col-xs-offset-0{margin-left:0} +@media (min-width:768px){ +.col-sm-1,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9{float:left} +.col-sm-12{width:100%} +.col-sm-11{width:91.66666667%} +.col-sm-10{width:83.33333333%} +.col-sm-9{width:75%} +.col-sm-8{width:66.66666667%} +.col-sm-7{width:58.33333333%} +.col-sm-6{width:50%} +.col-sm-5{width:41.66666667%} +.col-sm-4{width:33.33333333%} +.col-sm-3{width:25%} +.col-sm-2{width:16.66666667%} +.col-sm-1{width:8.33333333%} +.col-sm-pull-12{right:100%} +.col-sm-pull-11{right:91.66666667%} +.col-sm-pull-10{right:83.33333333%} +.col-sm-pull-9{right:75%} +.col-sm-pull-8{right:66.66666667%} +.col-sm-pull-7{right:58.33333333%} +.col-sm-pull-6{right:50%} +.col-sm-pull-5{right:41.66666667%} +.col-sm-pull-4{right:33.33333333%} +.col-sm-pull-3{right:25%} +.col-sm-pull-2{right:16.66666667%} +.col-sm-pull-1{right:8.33333333%} +.col-sm-pull-0{right:auto} +.col-sm-push-12{left:100%} +.col-sm-push-11{left:91.66666667%} +.col-sm-push-10{left:83.33333333%} +.col-sm-push-9{left:75%} +.col-sm-push-8{left:66.66666667%} +.col-sm-push-7{left:58.33333333%} +.col-sm-push-6{left:50%} +.col-sm-push-5{left:41.66666667%} +.col-sm-push-4{left:33.33333333%} +.col-sm-push-3{left:25%} +.col-sm-push-2{left:16.66666667%} +.col-sm-push-1{left:8.33333333%} +.col-sm-push-0{left:auto} +.col-sm-offset-12{margin-left:100%} +.col-sm-offset-11{margin-left:91.66666667%} +.col-sm-offset-10{margin-left:83.33333333%} +.col-sm-offset-9{margin-left:75%} +.col-sm-offset-8{margin-left:66.66666667%} +.col-sm-offset-7{margin-left:58.33333333%} +.col-sm-offset-6{margin-left:50%} +.col-sm-offset-5{margin-left:41.66666667%} +.col-sm-offset-4{margin-left:33.33333333%} +.col-sm-offset-3{margin-left:25%} +.col-sm-offset-2{margin-left:16.66666667%} +.col-sm-offset-1{margin-left:8.33333333%} +.col-sm-offset-0{margin-left:0} +} +@media (min-width:992px){ +.col-md-1,.col-md-10,.col-md-11,.col-md-12,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9{float:left} +.col-md-12{width:100%} +.col-md-11{width:91.66666667%} +.col-md-10{width:83.33333333%} +.col-md-9{width:75%} +.col-md-8{width:66.66666667%} +.col-md-7{width:58.33333333%} +.col-md-6{width:50%} +.col-md-5{width:41.66666667%} +.col-md-4{width:33.33333333%} +.col-md-3{width:25%} +.col-md-2{width:16.66666667%} +.col-md-1{width:8.33333333%} +.col-md-pull-12{right:100%} +.col-md-pull-11{right:91.66666667%} +.col-md-pull-10{right:83.33333333%} +.col-md-pull-9{right:75%} +.col-md-pull-8{right:66.66666667%} +.col-md-pull-7{right:58.33333333%} +.col-md-pull-6{right:50%} +.col-md-pull-5{right:41.66666667%} +.col-md-pull-4{right:33.33333333%} +.col-md-pull-3{right:25%} +.col-md-pull-2{right:16.66666667%} +.col-md-pull-1{right:8.33333333%} +.col-md-pull-0{right:auto} +.col-md-push-12{left:100%} +.col-md-push-11{left:91.66666667%} +.col-md-push-10{left:83.33333333%} +.col-md-push-9{left:75%} +.col-md-push-8{left:66.66666667%} +.col-md-push-7{left:58.33333333%} +.col-md-push-6{left:50%} +.col-md-push-5{left:41.66666667%} +.col-md-push-4{left:33.33333333%} +.col-md-push-3{left:25%} +.col-md-push-2{left:16.66666667%} +.col-md-push-1{left:8.33333333%} +.col-md-push-0{left:auto} +.col-md-offset-12{margin-left:100%} +.col-md-offset-11{margin-left:91.66666667%} +.col-md-offset-10{margin-left:83.33333333%} +.col-md-offset-9{margin-left:75%} +.col-md-offset-8{margin-left:66.66666667%} +.col-md-offset-7{margin-left:58.33333333%} +.col-md-offset-6{margin-left:50%} +.col-md-offset-5{margin-left:41.66666667%} +.col-md-offset-4{margin-left:33.33333333%} +.col-md-offset-3{margin-left:25%} +.col-md-offset-2{margin-left:16.66666667%} +.col-md-offset-1{margin-left:8.33333333%} +.col-md-offset-0{margin-left:0} +} +@media (min-width:1200px){ +.col-lg-1,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9{float:left} +.col-lg-12{width:100%} +.col-lg-11{width:91.66666667%} +.col-lg-10{width:83.33333333%} +.col-lg-9{width:75%} +.col-lg-8{width:66.66666667%} +.col-lg-7{width:58.33333333%} +.col-lg-6{width:50%} +.col-lg-5{width:41.66666667%} +.col-lg-4{width:33.33333333%} +.col-lg-3{width:25%} +.col-lg-2{width:16.66666667%} +.col-lg-1{width:8.33333333%} +.col-lg-pull-12{right:100%} +.col-lg-pull-11{right:91.66666667%} +.col-lg-pull-10{right:83.33333333%} +.col-lg-pull-9{right:75%} +.col-lg-pull-8{right:66.66666667%} +.col-lg-pull-7{right:58.33333333%} +.col-lg-pull-6{right:50%} +.col-lg-pull-5{right:41.66666667%} +.col-lg-pull-4{right:33.33333333%} +.col-lg-pull-3{right:25%} +.col-lg-pull-2{right:16.66666667%} +.col-lg-pull-1{right:8.33333333%} +.col-lg-pull-0{right:auto} +.col-lg-push-12{left:100%} +.col-lg-push-11{left:91.66666667%} +.col-lg-push-10{left:83.33333333%} +.col-lg-push-9{left:75%} +.col-lg-push-8{left:66.66666667%} +.col-lg-push-7{left:58.33333333%} +.col-lg-push-6{left:50%} +.col-lg-push-5{left:41.66666667%} +.col-lg-push-4{left:33.33333333%} +.col-lg-push-3{left:25%} +.col-lg-push-2{left:16.66666667%} +.col-lg-push-1{left:8.33333333%} +.col-lg-push-0{left:auto} +.col-lg-offset-12{margin-left:100%} +.col-lg-offset-11{margin-left:91.66666667%} +.col-lg-offset-10{margin-left:83.33333333%} +.col-lg-offset-9{margin-left:75%} +.col-lg-offset-8{margin-left:66.66666667%} +.col-lg-offset-7{margin-left:58.33333333%} +.col-lg-offset-6{margin-left:50%} +.col-lg-offset-5{margin-left:41.66666667%} +.col-lg-offset-4{margin-left:33.33333333%} +.col-lg-offset-3{margin-left:25%} +.col-lg-offset-2{margin-left:16.66666667%} +.col-lg-offset-1{margin-left:8.33333333%} +.col-lg-offset-0{margin-left:0} +} +table{border-collapse:collapse;border-spacing:0;background-color:transparent} +table col[class*=col-]{position:static;display:table-column;float:none} +table td[class*=col-],table th[class*=col-]{position:static;display:table-cell;float:none} +caption{padding-top:8px;padding-bottom:8px;color:#777;text-align:left} +th{text-align:left} +.table{width:100%;max-width:100%;margin-bottom:20px} +.table>tbody>tr>td,.table>tbody>tr>th,.table>tfoot>tr>td,.table>tfoot>tr>th,.table>thead>tr>td,.table>thead>tr>th{padding:8px;line-height:1.42857143;vertical-align:top;border-top:1px solid #ddd} +.table>thead>tr>th{vertical-align:bottom;border-bottom:2px solid #ddd} +.table>caption+thead>tr:first-child>td,.table>caption+thead>tr:first-child>th,.table>colgroup+thead>tr:first-child>td,.table>colgroup+thead>tr:first-child>th,.table>thead:first-child>tr:first-child>td,.table>thead:first-child>tr:first-child>th{border-top:0} +.table>tbody+tbody{border-top:2px solid #ddd} +.table .table{background-color:#fff} +.table-condensed>tbody>tr>td,.table-condensed>tbody>tr>th,.table-condensed>tfoot>tr>td,.table-condensed>tfoot>tr>th,.table-condensed>thead>tr>td,.table-condensed>thead>tr>th{padding:5px} +.table-bordered,.table-bordered>tbody>tr>td,.table-bordered>tbody>tr>th,.table-bordered>tfoot>tr>td,.table-bordered>tfoot>tr>th,.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border:1px solid #ddd} +.table-bordered>thead>tr>td,.table-bordered>thead>tr>th{border-bottom-width:2px} +.table-striped>tbody>tr:nth-of-type(odd){background-color:#f9f9f9} +.table-hover>tbody>tr:hover,.table>tbody>tr.active>td,.table>tbody>tr.active>th,.table>tbody>tr>td.active,.table>tbody>tr>th.active,.table>tfoot>tr.active>td,.table>tfoot>tr.active>th,.table>tfoot>tr>td.active,.table>tfoot>tr>th.active,.table>thead>tr.active>td,.table>thead>tr.active>th,.table>thead>tr>td.active,.table>thead>tr>th.active{background-color:#f5f5f5} +.table-hover>tbody>tr.active:hover>td,.table-hover>tbody>tr.active:hover>th,.table-hover>tbody>tr:hover>.active,.table-hover>tbody>tr>td.active:hover,.table-hover>tbody>tr>th.active:hover{background-color:#e8e8e8} +.table>tbody>tr.success>td,.table>tbody>tr.success>th,.table>tbody>tr>td.success,.table>tbody>tr>th.success,.table>tfoot>tr.success>td,.table>tfoot>tr.success>th,.table>tfoot>tr>td.success,.table>tfoot>tr>th.success,.table>thead>tr.success>td,.table>thead>tr.success>th,.table>thead>tr>td.success,.table>thead>tr>th.success{background-color:#dff0d8} +.table-hover>tbody>tr.success:hover>td,.table-hover>tbody>tr.success:hover>th,.table-hover>tbody>tr:hover>.success,.table-hover>tbody>tr>td.success:hover,.table-hover>tbody>tr>th.success:hover{background-color:#d0e9c6} +.table>tbody>tr.info>td,.table>tbody>tr.info>th,.table>tbody>tr>td.info,.table>tbody>tr>th.info,.table>tfoot>tr.info>td,.table>tfoot>tr.info>th,.table>tfoot>tr>td.info,.table>tfoot>tr>th.info,.table>thead>tr.info>td,.table>thead>tr.info>th,.table>thead>tr>td.info,.table>thead>tr>th.info{background-color:#d9edf7} +.table-hover>tbody>tr.info:hover>td,.table-hover>tbody>tr.info:hover>th,.table-hover>tbody>tr:hover>.info,.table-hover>tbody>tr>td.info:hover,.table-hover>tbody>tr>th.info:hover{background-color:#c4e3f3} +.table>tbody>tr.warning>td,.table>tbody>tr.warning>th,.table>tbody>tr>td.warning,.table>tbody>tr>th.warning,.table>tfoot>tr.warning>td,.table>tfoot>tr.warning>th,.table>tfoot>tr>td.warning,.table>tfoot>tr>th.warning,.table>thead>tr.warning>td,.table>thead>tr.warning>th,.table>thead>tr>td.warning,.table>thead>tr>th.warning{background-color:#fcf8e3} +.table-hover>tbody>tr.warning:hover>td,.table-hover>tbody>tr.warning:hover>th,.table-hover>tbody>tr:hover>.warning,.table-hover>tbody>tr>td.warning:hover,.table-hover>tbody>tr>th.warning:hover{background-color:#faf2cc} +.table>tbody>tr.danger>td,.table>tbody>tr.danger>th,.table>tbody>tr>td.danger,.table>tbody>tr>th.danger,.table>tfoot>tr.danger>td,.table>tfoot>tr.danger>th,.table>tfoot>tr>td.danger,.table>tfoot>tr>th.danger,.table>thead>tr.danger>td,.table>thead>tr.danger>th,.table>thead>tr>td.danger,.table>thead>tr>th.danger{background-color:#f2dede} +.table-hover>tbody>tr.danger:hover>td,.table-hover>tbody>tr.danger:hover>th,.table-hover>tbody>tr:hover>.danger,.table-hover>tbody>tr>td.danger:hover,.table-hover>tbody>tr>th.danger:hover{background-color:#ebcccc} +.table-responsive{min-height:.01%;overflow-x:auto} +@media screen and (max-width:767px){ +.table-responsive{width:100%;margin-bottom:15px;overflow-y:hidden;-ms-overflow-style:-ms-autohiding-scrollbar;border:1px solid #ddd} +.table-responsive>.table{margin-bottom:0} +.table-responsive>.table>tbody>tr>td,.table-responsive>.table>tbody>tr>th,.table-responsive>.table>tfoot>tr>td,.table-responsive>.table>tfoot>tr>th,.table-responsive>.table>thead>tr>td,.table-responsive>.table>thead>tr>th{white-space:nowrap} +.table-responsive>.table-bordered{border:0} +.table-responsive>.table-bordered>tbody>tr>td:first-child,.table-responsive>.table-bordered>tbody>tr>th:first-child,.table-responsive>.table-bordered>tfoot>tr>td:first-child,.table-responsive>.table-bordered>tfoot>tr>th:first-child,.table-responsive>.table-bordered>thead>tr>td:first-child,.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0} +.table-responsive>.table-bordered>tbody>tr>td:last-child,.table-responsive>.table-bordered>tbody>tr>th:last-child,.table-responsive>.table-bordered>tfoot>tr>td:last-child,.table-responsive>.table-bordered>tfoot>tr>th:last-child,.table-responsive>.table-bordered>thead>tr>td:last-child,.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0} +.table-responsive>.table-bordered>tbody>tr:last-child>td,.table-responsive>.table-bordered>tbody>tr:last-child>th,.table-responsive>.table-bordered>tfoot>tr:last-child>td,.table-responsive>.table-bordered>tfoot>tr:last-child>th{border-bottom:0} +} +fieldset{min-width:0;padding:0;margin:0;border:0} +legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:inherit;color:#333;border:0;border-bottom:1px solid #e5e5e5} +label{display:inline-block;max-width:100%;margin-bottom:5px;font-weight:700} +input[type=search]{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-appearance:none;-moz-appearance:none;appearance:none} +input[type=checkbox],input[type=radio]{margin:4px 0 0;line-height:normal} +fieldset[disabled] input[type=checkbox],fieldset[disabled] input[type=radio],input[type=checkbox].disabled,input[type=checkbox][disabled],input[type=radio].disabled,input[type=radio][disabled]{cursor:not-allowed} +input[type=file]{display:block} +input[type=range]{display:block;width:100%} +select[multiple],select[size]{height:auto} +input[type=checkbox]:focus,input[type=file]:focus,input[type=radio]:focus{outline:-webkit-focus-ring-color auto 5px;outline-offset:-2px} +output{display:block;padding-top:7px;font-size:14px;line-height:1.42857143;color:#555} +.form-control{display:block;width:100%;height:34px;padding:6px 12px;font-size:14px;line-height:1.42857143;color:#555;background-color:#fff;background-image:none;border:1px solid #ccc;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075);-webkit-transition:border-color .15s ease-in-out,-webkit-box-shadow .15s ease-in-out;-o-transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out,-webkit-box-shadow .15s ease-in-out} +.form-control:focus{border-color:#66afe9;outline:0;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6);box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 8px rgba(102,175,233,.6)} +.form-control::-moz-placeholder{color:#999;opacity:1} +.form-control:-ms-input-placeholder{color:#999} +.form-control::-webkit-input-placeholder{color:#999} +.form-control::-ms-expand{background-color:transparent;border:0} +.form-control[disabled],.form-control[readonly],fieldset[disabled] .form-control{background-color:#eee;opacity:1} +.form-control[disabled],fieldset[disabled] .form-control{cursor:not-allowed} +textarea.form-control{height:auto} +@media screen and (-webkit-min-device-pixel-ratio:0){ +input[type=date].form-control,input[type=datetime-local].form-control,input[type=month].form-control,input[type=time].form-control{line-height:34px} +.input-group-sm input[type=date],.input-group-sm input[type=datetime-local],.input-group-sm input[type=month],.input-group-sm input[type=time],input[type=date].input-sm,input[type=datetime-local].input-sm,input[type=month].input-sm,input[type=time].input-sm{line-height:30px} +.input-group-lg input[type=date],.input-group-lg input[type=datetime-local],.input-group-lg input[type=month],.input-group-lg input[type=time],input[type=date].input-lg,input[type=datetime-local].input-lg,input[type=month].input-lg,input[type=time].input-lg{line-height:46px} +} +.form-group{margin-bottom:15px} +.checkbox,.radio{position:relative;display:block;margin-top:10px;margin-bottom:10px} +.checkbox.disabled label,.radio.disabled label,fieldset[disabled] .checkbox label,fieldset[disabled] .radio label{cursor:not-allowed} +.checkbox label,.radio label{min-height:20px;padding-left:20px;margin-bottom:0;font-weight:400;cursor:pointer} +.checkbox input[type=checkbox],.checkbox-inline input[type=checkbox],.radio input[type=radio],.radio-inline input[type=radio]{position:absolute;margin-left:-20px} +.checkbox+.checkbox,.radio+.radio{margin-top:-5px} +.checkbox-inline,.radio-inline{position:relative;display:inline-block;padding-left:20px;margin-bottom:0;font-weight:400;vertical-align:middle;cursor:pointer} +.checkbox-inline.disabled,.radio-inline.disabled,fieldset[disabled] .checkbox-inline,fieldset[disabled] .radio-inline{cursor:not-allowed} +.checkbox-inline+.checkbox-inline,.radio-inline+.radio-inline{margin-top:0;margin-left:10px} +.form-control-static{min-height:34px;padding-top:7px;padding-bottom:7px;margin-bottom:0} +.form-control-static.input-lg,.form-control-static.input-sm{padding-right:0;padding-left:0} +.input-sm{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px} +select.input-sm{height:30px;line-height:30px} +select[multiple].input-sm,textarea.input-sm{height:auto} +.form-group-sm .form-control{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px} +.form-group-sm select.form-control{height:30px;line-height:30px} +.form-group-sm select[multiple].form-control,.form-group-sm textarea.form-control{height:auto} +.form-group-sm .form-control-static{height:30px;min-height:32px;padding:6px 10px;font-size:12px;line-height:1.5} +.input-lg{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px} +select.input-lg{height:46px;line-height:46px} +select[multiple].input-lg,textarea.input-lg{height:auto} +.form-group-lg .form-control{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px} +.form-group-lg select.form-control{height:46px;line-height:46px} +.form-group-lg select[multiple].form-control,.form-group-lg textarea.form-control{height:auto} +.form-group-lg .form-control-static{height:46px;min-height:38px;padding:11px 16px;font-size:18px;line-height:1.3333333} +.has-feedback{position:relative} +.has-feedback .form-control{padding-right:42.5px} +.form-control-feedback{position:absolute;top:0;right:0;z-index:2;display:block;width:34px;height:34px;line-height:34px;text-align:center;pointer-events:none} +.form-group-lg .form-control+.form-control-feedback,.input-group-lg+.form-control-feedback,.input-lg+.form-control-feedback{width:46px;height:46px;line-height:46px} +.form-group-sm .form-control+.form-control-feedback,.input-group-sm+.form-control-feedback,.input-sm+.form-control-feedback{width:30px;height:30px;line-height:30px} +.has-success .checkbox,.has-success .checkbox-inline,.has-success .control-label,.has-success .help-block,.has-success .radio,.has-success .radio-inline,.has-success.checkbox label,.has-success.checkbox-inline label,.has-success.radio label,.has-success.radio-inline label{color:#3c763d} +.has-success .form-control{border-color:#3c763d;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)} +.has-success .form-control:focus{border-color:#2b542c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #67b168} +.has-success .input-group-addon{color:#3c763d;background-color:#dff0d8;border-color:#3c763d} +.has-success .form-control-feedback{color:#3c763d} +.has-warning .checkbox,.has-warning .checkbox-inline,.has-warning .control-label,.has-warning .help-block,.has-warning .radio,.has-warning .radio-inline,.has-warning.checkbox label,.has-warning.checkbox-inline label,.has-warning.radio label,.has-warning.radio-inline label{color:#8a6d3b} +.has-warning .form-control{border-color:#8a6d3b;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)} +.has-warning .form-control:focus{border-color:#66512c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #c0a16b} +.has-warning .input-group-addon{color:#8a6d3b;background-color:#fcf8e3;border-color:#8a6d3b} +.has-warning .form-control-feedback{color:#8a6d3b} +.has-error .checkbox,.has-error .checkbox-inline,.has-error .control-label,.has-error .help-block,.has-error .radio,.has-error .radio-inline,.has-error.checkbox label,.has-error.checkbox-inline label,.has-error.radio label,.has-error.radio-inline label{color:#a94442} +.has-error .form-control{border-color:#a94442;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 1px rgba(0,0,0,.075)} +.has-error .form-control:focus{border-color:#843534;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483;box-shadow:inset 0 1px 1px rgba(0,0,0,.075),0 0 6px #ce8483} +.has-error .input-group-addon{color:#a94442;background-color:#f2dede;border-color:#a94442} +.has-error .form-control-feedback{color:#a94442} +.has-feedback label~.form-control-feedback{top:25px} +.has-feedback label.sr-only~.form-control-feedback{top:0} +.help-block{display:block;margin-top:5px;margin-bottom:10px;color:#737373} +.form-horizontal .checkbox,.form-horizontal .checkbox-inline,.form-horizontal .radio,.form-horizontal .radio-inline{padding-top:7px;margin-top:0;margin-bottom:0} +.form-horizontal .checkbox,.form-horizontal .radio{min-height:27px} +.form-horizontal .form-group{margin-right:-15px;margin-left:-15px} +.form-horizontal .has-feedback .form-control-feedback{right:15px} +@media (min-width:768px){ +.form-inline .form-group{display:inline-block;margin-bottom:0;vertical-align:middle} +.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle} +.form-inline .form-control-static{display:inline-block} +.form-inline .input-group{display:inline-table;vertical-align:middle} +.form-inline .input-group .form-control,.form-inline .input-group .input-group-addon,.form-inline .input-group .input-group-btn{width:auto} +.form-inline .input-group>.form-control{width:100%} +.form-inline .control-label{margin-bottom:0;vertical-align:middle} +.form-inline .checkbox,.form-inline .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle} +.form-inline .checkbox label,.form-inline .radio label{padding-left:0} +.form-inline .checkbox input[type=checkbox],.form-inline .radio input[type=radio]{position:relative;margin-left:0} +.form-inline .has-feedback .form-control-feedback{top:0} +.form-horizontal .control-label{padding-top:7px;margin-bottom:0;text-align:right} +.form-horizontal .form-group-lg .control-label{padding-top:11px;font-size:18px} +.form-horizontal .form-group-sm .control-label{padding-top:6px;font-size:12px} +} +.btn{display:inline-block;margin-bottom:0;font-weight:400;text-align:center;white-space:nowrap;vertical-align:middle;-ms-touch-action:manipulation;touch-action:manipulation;cursor:pointer;background-image:none;border:1px solid transparent;padding:6px 12px;font-size:14px;line-height:1.42857143;border-radius:4px;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none} +.btn.active.focus,.btn.active:focus,.btn.focus,.btn:active.focus,.btn:active:focus,.btn:focus{outline:-webkit-focus-ring-color auto 5px;outline-offset:-2px} +.btn.focus,.btn:focus,.btn:hover{color:#333;text-decoration:none} +.btn.active,.btn:active{background-image:none;outline:0;-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)} +.btn.disabled,.btn[disabled],fieldset[disabled] .btn{cursor:not-allowed;opacity:.65;-webkit-box-shadow:none;box-shadow:none} +a.btn.disabled,fieldset[disabled] a.btn{pointer-events:none} +.btn-default{color:#333;background-color:#fff;border-color:#ccc} +.btn-default.focus,.btn-default:focus{color:#333;background-color:#e6e6e6;border-color:#8c8c8c} +.btn-default:hover{color:#333;background-color:#e6e6e6;border-color:#adadad} +.btn-default.active,.btn-default:active,.open>.dropdown-toggle.btn-default{color:#333;background-color:#e6e6e6;background-image:none;border-color:#adadad} +.btn-default.active.focus,.btn-default.active:focus,.btn-default.active:hover,.btn-default:active.focus,.btn-default:active:focus,.btn-default:active:hover,.open>.dropdown-toggle.btn-default.focus,.open>.dropdown-toggle.btn-default:focus,.open>.dropdown-toggle.btn-default:hover{color:#333;background-color:#d4d4d4;border-color:#8c8c8c} +.btn-default.disabled.focus,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled].focus,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#fff;border-color:#ccc} +.btn-default .badge{color:#fff;background-color:#333} +.btn-primary{color:#fff;background-color:#337ab7;border-color:#2e6da4} +.btn-primary.focus,.btn-primary:focus{color:#fff;background-color:#286090;border-color:#122b40} +.btn-primary:hover{color:#fff;background-color:#286090;border-color:#204d74} +.btn-primary.active,.btn-primary:active,.open>.dropdown-toggle.btn-primary{color:#fff;background-color:#286090;background-image:none;border-color:#204d74} +.btn-primary.active.focus,.btn-primary.active:focus,.btn-primary.active:hover,.btn-primary:active.focus,.btn-primary:active:focus,.btn-primary:active:hover,.open>.dropdown-toggle.btn-primary.focus,.open>.dropdown-toggle.btn-primary:focus,.open>.dropdown-toggle.btn-primary:hover{color:#fff;background-color:#204d74;border-color:#122b40} +.btn-primary.disabled.focus,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled].focus,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#337ab7;border-color:#2e6da4} +.btn-primary .badge{color:#337ab7;background-color:#fff} +.btn-success{color:#fff;background-color:#5cb85c;border-color:#4cae4c} +.btn-success.focus,.btn-success:focus{color:#fff;background-color:#449d44;border-color:#255625} +.btn-success:hover{color:#fff;background-color:#449d44;border-color:#398439} +.btn-success.active,.btn-success:active,.open>.dropdown-toggle.btn-success{color:#fff;background-color:#449d44;background-image:none;border-color:#398439} +.btn-success.active.focus,.btn-success.active:focus,.btn-success.active:hover,.btn-success:active.focus,.btn-success:active:focus,.btn-success:active:hover,.open>.dropdown-toggle.btn-success.focus,.open>.dropdown-toggle.btn-success:focus,.open>.dropdown-toggle.btn-success:hover{color:#fff;background-color:#398439;border-color:#255625} +.btn-success.disabled.focus,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled].focus,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#5cb85c;border-color:#4cae4c} +.btn-success .badge{color:#5cb85c;background-color:#fff} +.btn-info{color:#fff;background-color:#5bc0de;border-color:#46b8da} +.btn-info.focus,.btn-info:focus{color:#fff;background-color:#31b0d5;border-color:#1b6d85} +.btn-info:hover{color:#fff;background-color:#31b0d5;border-color:#269abc} +.btn-info.active,.btn-info:active,.open>.dropdown-toggle.btn-info{color:#fff;background-color:#31b0d5;background-image:none;border-color:#269abc} +.btn-info.active.focus,.btn-info.active:focus,.btn-info.active:hover,.btn-info:active.focus,.btn-info:active:focus,.btn-info:active:hover,.open>.dropdown-toggle.btn-info.focus,.open>.dropdown-toggle.btn-info:focus,.open>.dropdown-toggle.btn-info:hover{color:#fff;background-color:#269abc;border-color:#1b6d85} +.btn-info.disabled.focus,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled].focus,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#5bc0de;border-color:#46b8da} +.btn-info .badge{color:#5bc0de;background-color:#fff} +.btn-warning{color:#fff;background-color:#f0ad4e;border-color:#eea236} +.btn-warning.focus,.btn-warning:focus{color:#fff;background-color:#ec971f;border-color:#985f0d} +.btn-warning:hover{color:#fff;background-color:#ec971f;border-color:#d58512} +.btn-warning.active,.btn-warning:active,.open>.dropdown-toggle.btn-warning{color:#fff;background-color:#ec971f;background-image:none;border-color:#d58512} +.btn-warning.active.focus,.btn-warning.active:focus,.btn-warning.active:hover,.btn-warning:active.focus,.btn-warning:active:focus,.btn-warning:active:hover,.open>.dropdown-toggle.btn-warning.focus,.open>.dropdown-toggle.btn-warning:focus,.open>.dropdown-toggle.btn-warning:hover{color:#fff;background-color:#d58512;border-color:#985f0d} +.btn-warning.disabled.focus,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled].focus,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#f0ad4e;border-color:#eea236} +.btn-warning .badge{color:#f0ad4e;background-color:#fff} +.btn-danger{color:#fff;background-color:#d9534f;border-color:#d43f3a} +.btn-danger.focus,.btn-danger:focus{color:#fff;background-color:#c9302c;border-color:#761c19} +.btn-danger:hover{color:#fff;background-color:#c9302c;border-color:#ac2925} +.btn-danger.active,.btn-danger:active,.open>.dropdown-toggle.btn-danger{color:#fff;background-color:#c9302c;background-image:none;border-color:#ac2925} +.btn-danger.active.focus,.btn-danger.active:focus,.btn-danger.active:hover,.btn-danger:active.focus,.btn-danger:active:focus,.btn-danger:active:hover,.open>.dropdown-toggle.btn-danger.focus,.open>.dropdown-toggle.btn-danger:focus,.open>.dropdown-toggle.btn-danger:hover{color:#fff;background-color:#ac2925;border-color:#761c19} +.btn-danger.disabled.focus,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled].focus,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#d9534f;border-color:#d43f3a} +.btn-danger .badge{color:#d9534f;background-color:#fff} +.btn-link{font-weight:400;color:#337ab7;border-radius:0} +.btn-link,.btn-link.active,.btn-link:active,.btn-link[disabled],fieldset[disabled] .btn-link{background-color:transparent;-webkit-box-shadow:none;box-shadow:none} +.btn-link,.btn-link:active,.btn-link:focus,.btn-link:hover{border-color:transparent} +.btn-link:focus,.btn-link:hover{color:#23527c;text-decoration:underline;background-color:transparent} +.btn-link[disabled]:focus,.btn-link[disabled]:hover,fieldset[disabled] .btn-link:focus,fieldset[disabled] .btn-link:hover{color:#777;text-decoration:none} +.btn-group-lg>.btn,.btn-lg{padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px} +.btn-group-sm>.btn,.btn-sm{padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px} +.btn-group-xs>.btn,.btn-xs{padding:1px 5px;font-size:12px;line-height:1.5;border-radius:3px} +.btn-block{display:block;width:100%} +.btn-block+.btn-block{margin-top:5px} +input[type=button].btn-block,input[type=reset].btn-block,input[type=submit].btn-block{width:100%} +.fade{opacity:0;-webkit-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear} +.fade.in{opacity:1} +.collapse{display:none} +.collapse.in{display:block} +tr.collapse.in{display:table-row} +tbody.collapse.in{display:table-row-group} +.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition-property:height,visibility;-o-transition-property:height,visibility;transition-property:height,visibility;-webkit-transition-duration:.35s;-o-transition-duration:.35s;transition-duration:.35s;-webkit-transition-timing-function:ease;-o-transition-timing-function:ease;transition-timing-function:ease} +.caret{display:inline-block;width:0;height:0;margin-left:2px;vertical-align:middle;border-top:4px dashed;border-right:4px solid transparent;border-left:4px solid transparent} +.dropdown,.dropup{position:relative} +.dropdown-toggle:focus{outline:0} +.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;font-size:14px;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:4px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175)} +.dropdown-menu.pull-right{right:0;left:auto} +.dropdown-menu .divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5} +.dropdown-menu>li>a{display:block;padding:3px 20px;clear:both;font-weight:400;line-height:1.42857143;color:#333;white-space:nowrap} +.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{color:#262626;text-decoration:none;background-color:#f5f5f5} +.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{color:#fff;text-decoration:none;background-color:#337ab7;outline:0} +.dropdown-menu>.disabled>a,.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{color:#777} +.dropdown-menu>.disabled>a:focus,.dropdown-menu>.disabled>a:hover{text-decoration:none;cursor:not-allowed;background-color:transparent;background-image:none} +.open>.dropdown-menu{display:block} +.open>a{outline:0} +.dropdown-menu-right{right:0;left:auto} +.dropdown-menu-left{right:auto;left:0} +.dropdown-header{display:block;padding:3px 20px;font-size:12px;line-height:1.42857143;color:#777;white-space:nowrap} +.dropdown-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:990} +.pull-right>.dropdown-menu{right:0;left:auto} +.dropup .caret,.navbar-fixed-bottom .dropdown .caret{content:"";border-top:0;border-bottom:4px dashed} +.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:2px} +.btn-group,.btn-group-vertical{position:relative;display:inline-block;vertical-align:middle} +.btn-group-vertical>.btn,.btn-group>.btn{position:relative;float:left} +.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:2} +.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group{margin-left:-1px} +.btn-toolbar{margin-left:-5px} +.btn-toolbar .btn,.btn-toolbar .btn-group,.btn-toolbar .input-group{float:left} +.btn-toolbar>.btn,.btn-toolbar>.btn-group,.btn-toolbar>.input-group{margin-left:5px} +.btn-group>.btn:not(:first-child):not(:last-child):not(.dropdown-toggle){border-radius:0} +.btn-group>.btn:first-child{margin-left:0} +.btn-group>.btn:first-child:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0} +.btn-group>.btn:last-child:not(:first-child),.btn-group>.dropdown-toggle:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0} +.btn-group>.btn-group{float:left} +.btn-group>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0} +.btn-group>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-top-right-radius:0;border-bottom-right-radius:0} +.btn-group>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-bottom-left-radius:0} +.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0} +.btn-group>.btn+.dropdown-toggle{padding-right:8px;padding-left:8px} +.btn-group>.btn-lg+.dropdown-toggle{padding-right:12px;padding-left:12px} +.btn-group.open .dropdown-toggle{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)} +.btn-group.open .dropdown-toggle.btn-link{-webkit-box-shadow:none;box-shadow:none} +.btn .caret{margin-left:0} +.btn-lg .caret{border-width:5px 5px 0} +.dropup .btn-lg .caret{border-width:0 5px 5px} +.btn-group-vertical>.btn,.btn-group-vertical>.btn-group,.btn-group-vertical>.btn-group>.btn{display:block;float:none;width:100%;max-width:100%} +.btn-group-vertical>.btn-group>.btn{float:none} +.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0} +.btn-group-vertical>.btn:not(:first-child):not(:last-child){border-radius:0} +.btn-group-vertical>.btn:first-child:not(:last-child){border-radius:4px 4px 0 0} +.btn-group-vertical>.btn:last-child:not(:first-child){border-radius:0 0 4px 4px} +.btn-group-vertical>.btn-group:not(:first-child):not(:last-child)>.btn{border-radius:0} +.btn-group-vertical>.btn-group:first-child:not(:last-child)>.btn:last-child,.btn-group-vertical>.btn-group:first-child:not(:last-child)>.dropdown-toggle{border-bottom-right-radius:0;border-bottom-left-radius:0} +.btn-group-vertical>.btn-group:last-child:not(:first-child)>.btn:first-child{border-top-left-radius:0;border-top-right-radius:0} +.btn-group-justified{display:table;width:100%;table-layout:fixed;border-collapse:separate} +.btn-group-justified>.btn,.btn-group-justified>.btn-group{display:table-cell;float:none;width:1%} +.btn-group-justified>.btn-group .btn{width:100%} +.btn-group-justified>.btn-group .dropdown-menu{left:auto} +[data-toggle=buttons]>.btn input[type=checkbox],[data-toggle=buttons]>.btn input[type=radio],[data-toggle=buttons]>.btn-group>.btn input[type=checkbox],[data-toggle=buttons]>.btn-group>.btn input[type=radio]{position:absolute;clip:rect(0,0,0,0);pointer-events:none} +.input-group{position:relative;display:table;border-collapse:separate} +.input-group[class*=col-]{float:none;padding-right:0;padding-left:0} +.input-group .form-control{position:relative;z-index:2;float:left;width:100%;margin-bottom:0} +.input-group .form-control:focus{z-index:3} +.input-group-lg>.form-control,.input-group-lg>.input-group-addon,.input-group-lg>.input-group-btn>.btn{height:46px;padding:10px 16px;font-size:18px;line-height:1.3333333;border-radius:6px} +select.input-group-lg>.form-control,select.input-group-lg>.input-group-addon,select.input-group-lg>.input-group-btn>.btn{height:46px;line-height:46px} +select[multiple].input-group-lg>.form-control,select[multiple].input-group-lg>.input-group-addon,select[multiple].input-group-lg>.input-group-btn>.btn,textarea.input-group-lg>.form-control,textarea.input-group-lg>.input-group-addon,textarea.input-group-lg>.input-group-btn>.btn{height:auto} +.input-group-sm>.form-control,.input-group-sm>.input-group-addon,.input-group-sm>.input-group-btn>.btn{height:30px;padding:5px 10px;font-size:12px;line-height:1.5;border-radius:3px} +select.input-group-sm>.form-control,select.input-group-sm>.input-group-addon,select.input-group-sm>.input-group-btn>.btn{height:30px;line-height:30px} +select[multiple].input-group-sm>.form-control,select[multiple].input-group-sm>.input-group-addon,select[multiple].input-group-sm>.input-group-btn>.btn,textarea.input-group-sm>.form-control,textarea.input-group-sm>.input-group-addon,textarea.input-group-sm>.input-group-btn>.btn{height:auto} +.input-group .form-control,.input-group-addon,.input-group-btn{display:table-cell} +.input-group .form-control:not(:first-child):not(:last-child),.input-group-addon:not(:first-child):not(:last-child),.input-group-btn:not(:first-child):not(:last-child){border-radius:0} +.input-group-addon,.input-group-btn{width:1%;white-space:nowrap;vertical-align:middle} +.input-group-addon{padding:6px 12px;font-size:14px;font-weight:400;line-height:1;color:#555;text-align:center;background-color:#eee;border:1px solid #ccc;border-radius:4px} +.input-group-addon.input-sm{padding:5px 10px;font-size:12px;border-radius:3px} +.input-group-addon.input-lg{padding:10px 16px;font-size:18px;border-radius:6px} +.input-group-addon input[type=checkbox],.input-group-addon input[type=radio]{margin-top:0} +.input-group .form-control:first-child,.input-group-addon:first-child,.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group>.btn,.input-group-btn:first-child>.dropdown-toggle,.input-group-btn:last-child>.btn-group:not(:last-child)>.btn,.input-group-btn:last-child>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0} +.input-group-addon:first-child{border-right:0} +.input-group .form-control:last-child,.input-group-addon:last-child,.input-group-btn:first-child>.btn-group:not(:first-child)>.btn,.input-group-btn:first-child>.btn:not(:first-child),.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group>.btn,.input-group-btn:last-child>.dropdown-toggle{border-top-left-radius:0;border-bottom-left-radius:0} +.input-group-addon:last-child{border-left:0} +.input-group-btn{position:relative;font-size:0;white-space:nowrap} +.input-group-btn>.btn{position:relative} +.input-group-btn>.btn+.btn{margin-left:-1px} +.input-group-btn>.btn:active,.input-group-btn>.btn:focus,.input-group-btn>.btn:hover{z-index:2} +.input-group-btn:first-child>.btn,.input-group-btn:first-child>.btn-group{margin-right:-1px} +.input-group-btn:last-child>.btn,.input-group-btn:last-child>.btn-group{z-index:2;margin-left:-1px} +.nav{padding-left:0;margin-bottom:0;list-style:none} +.nav>li{position:relative;display:block} +.nav>li>a{position:relative;display:block;padding:10px 15px} +.nav>li>a:focus,.nav>li>a:hover{text-decoration:none;background-color:#eee} +.nav>li.disabled>a{color:#777} +.nav>li.disabled>a:focus,.nav>li.disabled>a:hover{color:#777;text-decoration:none;cursor:not-allowed;background-color:transparent} +.nav .open>a,.nav .open>a:focus,.nav .open>a:hover{background-color:#eee;border-color:#337ab7} +.nav .nav-divider{height:1px;margin:9px 0;overflow:hidden;background-color:#e5e5e5} +.nav>li>a>img{max-width:none} +.nav-tabs{border-bottom:1px solid #ddd} +.nav-tabs>li{float:left;margin-bottom:-1px} +.nav-tabs>li>a{margin-right:2px;line-height:1.42857143;border:1px solid transparent;border-radius:4px 4px 0 0} +.nav-tabs>li>a:hover{border-color:#eee #eee #ddd} +.nav-tabs>li.active>a,.nav-tabs>li.active>a:focus,.nav-tabs>li.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent} +.nav-tabs.nav-justified{width:100%;border-bottom:0} +.nav-tabs.nav-justified>li{float:none} +.nav-tabs.nav-justified>li>a{margin-bottom:5px;text-align:center;margin-right:0;border-radius:4px} +.nav-tabs.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto} +.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border:1px solid #ddd} +.nav-pills>li{float:left} +.nav-pills>li>a{border-radius:4px} +.nav-pills>li+li{margin-left:2px} +.nav-pills>li.active>a,.nav-pills>li.active>a:focus,.nav-pills>li.active>a:hover{color:#fff;background-color:#337ab7} +.nav-stacked>li{float:none} +.nav-stacked>li+li{margin-top:2px;margin-left:0} +.nav-justified{width:100%} +.nav-justified>li{float:none} +.nav-justified>li>a{margin-bottom:5px;text-align:center} +.nav-justified>.dropdown .dropdown-menu{top:auto;left:auto} +@media (min-width:768px){ +.navbar-right .dropdown-menu{right:0;left:auto} +.navbar-right .dropdown-menu-left{right:auto;left:0} +.nav-tabs.nav-justified>li{display:table-cell;width:1%} +.nav-tabs.nav-justified>li>a{margin-bottom:0;border-bottom:1px solid #ddd;border-radius:4px 4px 0 0} +.nav-tabs.nav-justified>.active>a,.nav-tabs.nav-justified>.active>a:focus,.nav-tabs.nav-justified>.active>a:hover{border-bottom-color:#fff} +.nav-justified>li{display:table-cell;width:1%} +.nav-justified>li>a{margin-bottom:0} +} +.nav-tabs-justified{border-bottom:0} +.nav-tabs-justified>li>a{margin-right:0;border-radius:4px} +.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border:1px solid #ddd} +@media (min-width:768px){ +.nav-tabs-justified>li>a{border-bottom:1px solid #ddd;border-radius:4px 4px 0 0} +.nav-tabs-justified>.active>a,.nav-tabs-justified>.active>a:focus,.nav-tabs-justified>.active>a:hover{border-bottom-color:#fff} +} +.tab-content>.tab-pane{display:none} +.tab-content>.active{display:block} +.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0} +.navbar{position:relative;min-height:50px;margin-bottom:20px;border:1px solid transparent} +.navbar-collapse{padding-right:15px;padding-left:15px;overflow-x:visible;border-top:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1);-webkit-overflow-scrolling:touch} +.navbar-collapse.in{overflow-y:auto} +.navbar-fixed-bottom,.navbar-fixed-top{position:fixed;right:0;left:0;z-index:1030} +.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:340px} +@media (max-device-width:480px) and (orientation:landscape){ +.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse{max-height:200px} +} +@media (min-width:768px){ +.navbar{border-radius:4px} +.navbar-header{float:left} +.navbar-collapse{width:auto;border-top:0;-webkit-box-shadow:none;box-shadow:none} +.navbar-collapse.collapse{display:block!important;height:auto!important;padding-bottom:0;overflow:visible!important} +.navbar-collapse.in{overflow-y:visible} +.navbar-fixed-bottom .navbar-collapse,.navbar-fixed-top .navbar-collapse,.navbar-static-top .navbar-collapse{padding-right:0;padding-left:0} +.navbar-fixed-bottom,.navbar-fixed-top{border-radius:0} +} +.navbar-fixed-top{top:0;border-width:0 0 1px} +.navbar-fixed-bottom{bottom:0;margin-bottom:0;border-width:1px 0 0} +.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:-15px;margin-left:-15px} +.navbar-static-top{z-index:1000;border-width:0 0 1px} +.navbar-brand{float:left;height:50px;padding:15px;font-size:18px;line-height:20px} +.navbar-brand:focus,.navbar-brand:hover{text-decoration:none} +.navbar-brand>img{display:block} +@media (min-width:768px){ +.container-fluid>.navbar-collapse,.container-fluid>.navbar-header,.container>.navbar-collapse,.container>.navbar-header{margin-right:0;margin-left:0} +.navbar-static-top{border-radius:0} +.navbar>.container .navbar-brand,.navbar>.container-fluid .navbar-brand{margin-left:-15px} +.navbar-toggle{display:none} +} +.navbar-toggle{position:relative;float:right;padding:9px 10px;margin-right:15px;margin-top:8px;margin-bottom:8px;background-color:transparent;background-image:none;border:1px solid transparent;border-radius:4px} +.navbar-toggle:focus{outline:0} +.navbar-toggle .icon-bar{display:block;width:22px;height:2px;border-radius:1px} +.navbar-toggle .icon-bar+.icon-bar{margin-top:4px} +.navbar-nav{margin:7.5px -15px} +.navbar-nav>li>a{padding-top:10px;padding-bottom:10px;line-height:20px} +@media (max-width:767px){ +.navbar-nav .open .dropdown-menu{position:static;float:none;width:auto;margin-top:0;background-color:transparent;border:0;-webkit-box-shadow:none;box-shadow:none} +.navbar-nav .open .dropdown-menu .dropdown-header,.navbar-nav .open .dropdown-menu>li>a{padding:5px 15px 5px 25px} +.navbar-nav .open .dropdown-menu>li>a{line-height:20px} +.navbar-nav .open .dropdown-menu>li>a:focus,.navbar-nav .open .dropdown-menu>li>a:hover{background-image:none} +} +@media (min-width:768px){ +.navbar-nav{float:left;margin:0} +.navbar-nav>li{float:left} +.navbar-nav>li>a{padding-top:15px;padding-bottom:15px} +.navbar-form .form-group{display:inline-block;margin-bottom:0;vertical-align:middle} +.navbar-form .form-control{display:inline-block;width:auto;vertical-align:middle} +.navbar-form .form-control-static{display:inline-block} +.navbar-form .input-group{display:inline-table;vertical-align:middle} +.navbar-form .input-group .form-control,.navbar-form .input-group .input-group-addon,.navbar-form .input-group .input-group-btn{width:auto} +.navbar-form .input-group>.form-control{width:100%} +.navbar-form .control-label{margin-bottom:0;vertical-align:middle} +.navbar-form .checkbox,.navbar-form .radio{display:inline-block;margin-top:0;margin-bottom:0;vertical-align:middle} +.navbar-form .checkbox label,.navbar-form .radio label{padding-left:0} +.navbar-form .checkbox input[type=checkbox],.navbar-form .radio input[type=radio]{position:relative;margin-left:0} +.navbar-form .has-feedback .form-control-feedback{top:0} +} +.navbar-form{padding:10px 15px;border-top:1px solid transparent;border-bottom:1px solid transparent;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 0 rgba(255,255,255,.1),0 1px 0 rgba(255,255,255,.1);margin:8px -15px} +@media (max-width:767px){ +.navbar-form .form-group{margin-bottom:5px} +.navbar-form .form-group:last-child{margin-bottom:0} +.navbar-default .navbar-nav .open .dropdown-menu>li>a{color:#777} +.navbar-default .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>li>a:hover{color:#333;background-color:transparent} +.navbar-default .navbar-nav .open .dropdown-menu>.active>a,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.active>a:hover{color:#555;background-color:#e7e7e7} +.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-default .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#ccc;background-color:transparent} +} +@media (min-width:768px){ +.navbar-form{width:auto;padding-top:0;padding-bottom:0;margin-right:0;margin-left:0;border:0;-webkit-box-shadow:none;box-shadow:none} +.navbar-text{float:left;margin-right:15px;margin-left:15px} +} +.navbar-nav>li>.dropdown-menu{margin-top:0;border-top-left-radius:0;border-top-right-radius:0} +.navbar-fixed-bottom .navbar-nav>li>.dropdown-menu{margin-bottom:0;border-radius:4px 4px 0 0} +.navbar-btn{margin-top:8px;margin-bottom:8px} +.navbar-btn.btn-sm{margin-top:10px;margin-bottom:10px} +.navbar-btn.btn-xs{margin-top:14px;margin-bottom:14px} +.navbar-text{margin-top:15px;margin-bottom:15px} +@media (min-width:768px){ +.navbar-left{float:left!important} +.navbar-right{float:right!important;margin-right:-15px} +.navbar-right~.navbar-right{margin-right:0} +} +.navbar-default{background-color:#f8f8f8;border-color:#e7e7e7} +.navbar-default .navbar-brand{color:#777} +.navbar-default .navbar-brand:focus,.navbar-default .navbar-brand:hover{color:#5e5e5e;background-color:transparent} +.navbar-default .navbar-nav>li>a,.navbar-default .navbar-text{color:#777} +.navbar-default .navbar-nav>li>a:focus,.navbar-default .navbar-nav>li>a:hover{color:#333;background-color:transparent} +.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.active>a:focus,.navbar-default .navbar-nav>.active>a:hover{color:#555;background-color:#e7e7e7} +.navbar-default .navbar-nav>.disabled>a,.navbar-default .navbar-nav>.disabled>a:focus,.navbar-default .navbar-nav>.disabled>a:hover{color:#ccc;background-color:transparent} +.navbar-default .navbar-nav>.open>a,.navbar-default .navbar-nav>.open>a:focus,.navbar-default .navbar-nav>.open>a:hover{color:#555;background-color:#e7e7e7} +.navbar-default .navbar-toggle{border-color:#ddd} +.navbar-default .navbar-toggle:focus,.navbar-default .navbar-toggle:hover{background-color:#ddd} +.navbar-default .navbar-toggle .icon-bar{background-color:#888} +.navbar-default .navbar-collapse,.navbar-default .navbar-form{border-color:#e7e7e7} +.navbar-default .navbar-link{color:#777} +.navbar-default .navbar-link:hover{color:#333} +.navbar-default .btn-link{color:#777} +.navbar-default .btn-link:focus,.navbar-default .btn-link:hover{color:#333} +.navbar-default .btn-link[disabled]:focus,.navbar-default .btn-link[disabled]:hover,fieldset[disabled] .navbar-default .btn-link:focus,fieldset[disabled] .navbar-default .btn-link:hover{color:#ccc} +.navbar-inverse{background-color:#222;border-color:#080808} +.navbar-inverse .navbar-brand{color:#9d9d9d} +.navbar-inverse .navbar-brand:focus,.navbar-inverse .navbar-brand:hover{color:#fff;background-color:transparent} +.navbar-inverse .navbar-nav>li>a,.navbar-inverse .navbar-text{color:#9d9d9d} +.navbar-inverse .navbar-nav>li>a:focus,.navbar-inverse .navbar-nav>li>a:hover{color:#fff;background-color:transparent} +.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.active>a:focus,.navbar-inverse .navbar-nav>.active>a:hover{color:#fff;background-color:#080808} +.navbar-inverse .navbar-nav>.disabled>a,.navbar-inverse .navbar-nav>.disabled>a:focus,.navbar-inverse .navbar-nav>.disabled>a:hover{color:#444;background-color:transparent} +.navbar-inverse .navbar-nav>.open>a,.navbar-inverse .navbar-nav>.open>a:focus,.navbar-inverse .navbar-nav>.open>a:hover{color:#fff;background-color:#080808} +@media (max-width:767px){ +.navbar-inverse .navbar-nav .open .dropdown-menu>.dropdown-header{border-color:#080808} +.navbar-inverse .navbar-nav .open .dropdown-menu .divider{background-color:#080808} +.navbar-inverse .navbar-nav .open .dropdown-menu>li>a{color:#9d9d9d} +.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>li>a:hover{color:#fff;background-color:transparent} +.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-color:#080808} +.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:focus,.navbar-inverse .navbar-nav .open .dropdown-menu>.disabled>a:hover{color:#444;background-color:transparent} +} +.navbar-inverse .navbar-toggle{border-color:#333} +.navbar-inverse .navbar-toggle:focus,.navbar-inverse .navbar-toggle:hover{background-color:#333} +.navbar-inverse .navbar-toggle .icon-bar{background-color:#fff} +.navbar-inverse .navbar-collapse,.navbar-inverse .navbar-form{border-color:#101010} +.navbar-inverse .navbar-link{color:#9d9d9d} +.navbar-inverse .navbar-link:hover{color:#fff} +.navbar-inverse .btn-link{color:#9d9d9d} +.navbar-inverse .btn-link:focus,.navbar-inverse .btn-link:hover{color:#fff} +.navbar-inverse .btn-link[disabled]:focus,.navbar-inverse .btn-link[disabled]:hover,fieldset[disabled] .navbar-inverse .btn-link:focus,fieldset[disabled] .navbar-inverse .btn-link:hover{color:#444} +.breadcrumb{padding:8px 15px;margin-bottom:20px;list-style:none;background-color:#f5f5f5;border-radius:4px} +.breadcrumb>li{display:inline-block} +.breadcrumb>li+li:before{padding:0 5px;color:#ccc;content:"/\00a0"} +.breadcrumb>.active{color:#777} +.pagination{display:inline-block;padding-left:0;margin:20px 0;border-radius:4px} +.pagination>li{display:inline} +.pagination>li>a,.pagination>li>span{position:relative;float:left;padding:6px 12px;margin-left:-1px;line-height:1.42857143;color:#337ab7;text-decoration:none;background-color:#fff;border:1px solid #ddd} +.pagination>li>a:focus,.pagination>li>a:hover,.pagination>li>span:focus,.pagination>li>span:hover{z-index:2;color:#23527c;background-color:#eee;border-color:#ddd} +.pagination>li:first-child>a,.pagination>li:first-child>span{margin-left:0;border-top-left-radius:4px;border-bottom-left-radius:4px} +.pagination>li:last-child>a,.pagination>li:last-child>span{border-top-right-radius:4px;border-bottom-right-radius:4px} +.pagination>.active>a,.pagination>.active>a:focus,.pagination>.active>a:hover,.pagination>.active>span,.pagination>.active>span:focus,.pagination>.active>span:hover{z-index:3;color:#fff;cursor:default;background-color:#337ab7;border-color:#337ab7} +.pagination>.disabled>a,.pagination>.disabled>a:focus,.pagination>.disabled>a:hover,.pagination>.disabled>span,.pagination>.disabled>span:focus,.pagination>.disabled>span:hover{color:#777;cursor:not-allowed;background-color:#fff;border-color:#ddd} +.pagination-lg>li>a,.pagination-lg>li>span{padding:10px 16px;font-size:18px;line-height:1.3333333} +.pagination-lg>li:first-child>a,.pagination-lg>li:first-child>span{border-top-left-radius:6px;border-bottom-left-radius:6px} +.pagination-lg>li:last-child>a,.pagination-lg>li:last-child>span{border-top-right-radius:6px;border-bottom-right-radius:6px} +.pagination-sm>li>a,.pagination-sm>li>span{padding:5px 10px;font-size:12px;line-height:1.5} +.pagination-sm>li:first-child>a,.pagination-sm>li:first-child>span{border-top-left-radius:3px;border-bottom-left-radius:3px} +.pagination-sm>li:last-child>a,.pagination-sm>li:last-child>span{border-top-right-radius:3px;border-bottom-right-radius:3px} +.pager{padding-left:0;margin:20px 0;text-align:center;list-style:none} +.pager li{display:inline} +.pager li>a,.pager li>span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;border-radius:15px} +.pager li>a:focus,.pager li>a:hover{text-decoration:none;background-color:#eee} +.pager .next>a,.pager .next>span{float:right} +.pager .previous>a,.pager .previous>span{float:left} +.pager .disabled>a,.pager .disabled>a:focus,.pager .disabled>a:hover,.pager .disabled>span{color:#777;cursor:not-allowed;background-color:#fff} +.label{display:inline;padding:.2em .6em .3em;font-size:75%;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25em} +a.label:focus,a.label:hover{color:#fff;text-decoration:none;cursor:pointer} +.label:empty{display:none} +.btn .label{position:relative;top:-1px} +.label-default{background-color:#777} +.label-default[href]:focus,.label-default[href]:hover{background-color:#5e5e5e} +.label-primary{background-color:#337ab7} +.label-primary[href]:focus,.label-primary[href]:hover{background-color:#286090} +.label-success{background-color:#5cb85c} +.label-success[href]:focus,.label-success[href]:hover{background-color:#449d44} +.label-info{background-color:#5bc0de} +.label-info[href]:focus,.label-info[href]:hover{background-color:#31b0d5} +.label-warning{background-color:#f0ad4e} +.label-warning[href]:focus,.label-warning[href]:hover{background-color:#ec971f} +.label-danger{background-color:#d9534f} +.label-danger[href]:focus,.label-danger[href]:hover{background-color:#c9302c} +.badge{display:inline-block;min-width:10px;padding:3px 7px;font-size:12px;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:middle;background-color:#777;border-radius:10px} +.badge:empty{display:none} +.btn .badge{position:relative;top:-1px} +.btn-group-xs>.btn .badge,.btn-xs .badge{top:0;padding:1px 5px} +a.badge:focus,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer} +.list-group-item.active>.badge,.nav-pills>.active>a>.badge{color:#337ab7;background-color:#fff} +.list-group-item>.badge{float:right} +.list-group-item>.badge+.badge{margin-right:5px} +.nav-pills>li>a>.badge{margin-left:3px} +.jumbotron{padding-top:30px;padding-bottom:30px;margin-bottom:30px;color:inherit;background-color:#eee} +.jumbotron .h1,.jumbotron h1{color:inherit} +.jumbotron p{margin-bottom:15px;font-size:21px;font-weight:200} +.jumbotron>hr{border-top-color:#d5d5d5} +.container .jumbotron,.container-fluid .jumbotron{padding-right:15px;padding-left:15px;border-radius:6px} +.jumbotron .container{max-width:100%} +@media screen and (min-width:768px){ +.jumbotron{padding-top:48px;padding-bottom:48px} +.container .jumbotron,.container-fluid .jumbotron{padding-right:60px;padding-left:60px} +.jumbotron .h1,.jumbotron h1{font-size:63px} +} +.thumbnail{display:block;padding:4px;margin-bottom:20px;line-height:1.42857143;background-color:#fff;border:1px solid #ddd;border-radius:4px;-webkit-transition:border .2s ease-in-out;-o-transition:border .2s ease-in-out;transition:border .2s ease-in-out} +.thumbnail a>img,.thumbnail>img{margin-right:auto;margin-left:auto} +a.thumbnail.active,a.thumbnail:focus,a.thumbnail:hover{border-color:#337ab7} +.thumbnail .caption{padding:9px;color:#333} +.alert{padding:15px;margin-bottom:20px;border:1px solid transparent;border-radius:4px} +.alert h4{margin-top:0;color:inherit} +.alert .alert-link{font-weight:700} +.alert>p,.alert>ul{margin-bottom:0} +.alert>p+p{margin-top:5px} +.alert-dismissable,.alert-dismissible{padding-right:35px} +.alert-dismissable .close,.alert-dismissible .close{position:relative;top:-2px;right:-21px;color:inherit} +.alert-success{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6} +.alert-success hr{border-top-color:#c9e2b3} +.alert-success .alert-link{color:#2b542c} +.alert-info{color:#31708f;background-color:#d9edf7;border-color:#bce8f1} +.alert-info hr{border-top-color:#a6e1ec} +.alert-info .alert-link{color:#245269} +.alert-warning{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc} +.alert-warning hr{border-top-color:#f7e1b5} +.alert-warning .alert-link{color:#66512c} +.alert-danger{color:#a94442;background-color:#f2dede;border-color:#ebccd1} +.alert-danger hr{border-top-color:#e4b9c0} +.alert-danger .alert-link{color:#843534} +@-webkit-keyframes progress-bar-stripes{ +from{background-position:40px 0} +to{background-position:0 0} +} +@-o-keyframes progress-bar-stripes{ +from{background-position:40px 0} +to{background-position:0 0} +} +@keyframes progress-bar-stripes{ +from{background-position:40px 0} +to{background-position:0 0} +} +.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f5f5f5;border-radius:4px;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,.1);box-shadow:inset 0 1px 2px rgba(0,0,0,.1)} +.progress-bar{float:left;width:0%;height:100%;font-size:12px;line-height:20px;color:#fff;text-align:center;background-color:#337ab7;-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,.15);-webkit-transition:width .6s;-o-transition:width .6s;transition:width .6s} +.progress-bar-striped,.progress-striped .progress-bar{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;background-size:40px 40px} +.progress-bar.active,.progress.active .progress-bar{-webkit-animation:2s linear infinite progress-bar-stripes;-o-animation:2s linear infinite progress-bar-stripes;animation:2s linear infinite progress-bar-stripes} +.progress-bar-success{background-color:#5cb85c} +.progress-striped .progress-bar-success{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)} +.progress-bar-info{background-color:#5bc0de} +.progress-striped .progress-bar-info{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)} +.progress-bar-warning{background-color:#f0ad4e} +.progress-striped .progress-bar-warning{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)} +.progress-bar-danger{background-color:#d9534f} +.progress-striped .progress-bar-danger{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)} +.media{margin-top:15px} +.media:first-child{margin-top:0} +.media,.media-body{overflow:hidden;zoom:1} +.media-body{width:10000px} +.media-object{display:block} +.media-object.img-thumbnail{max-width:none} +.media-right,.media>.pull-right{padding-left:10px} +.media-left,.media>.pull-left{padding-right:10px} +.media-body,.media-left,.media-right{display:table-cell;vertical-align:top} +.media-middle{vertical-align:middle} +.media-bottom{vertical-align:bottom} +.media-heading{margin-top:0;margin-bottom:5px} +.media-list{padding-left:0;list-style:none} +.list-group{padding-left:0;margin-bottom:20px} +.list-group-item{position:relative;display:block;padding:10px 15px;margin-bottom:-1px;background-color:#fff;border:1px solid #ddd} +.list-group-item:first-child{border-top-left-radius:4px;border-top-right-radius:4px} +.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:4px;border-bottom-left-radius:4px} +.list-group-item.disabled,.list-group-item.disabled:focus,.list-group-item.disabled:hover{color:#777;cursor:not-allowed;background-color:#eee} +.list-group-item.disabled .list-group-item-heading,.list-group-item.disabled:focus .list-group-item-heading,.list-group-item.disabled:hover .list-group-item-heading{color:inherit} +.list-group-item.disabled .list-group-item-text,.list-group-item.disabled:focus .list-group-item-text,.list-group-item.disabled:hover .list-group-item-text{color:#777} +.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{z-index:2;color:#fff;background-color:#337ab7;border-color:#337ab7} +.list-group-item.active .list-group-item-heading,.list-group-item.active .list-group-item-heading>.small,.list-group-item.active .list-group-item-heading>small,.list-group-item.active:focus .list-group-item-heading,.list-group-item.active:focus .list-group-item-heading>.small,.list-group-item.active:focus .list-group-item-heading>small,.list-group-item.active:hover .list-group-item-heading,.list-group-item.active:hover .list-group-item-heading>.small,.list-group-item.active:hover .list-group-item-heading>small{color:inherit} +.list-group-item.active .list-group-item-text,.list-group-item.active:focus .list-group-item-text,.list-group-item.active:hover .list-group-item-text{color:#c7ddef} +a.list-group-item,button.list-group-item{color:#555} +a.list-group-item .list-group-item-heading,button.list-group-item .list-group-item-heading{color:#333} +a.list-group-item:focus,a.list-group-item:hover,button.list-group-item:focus,button.list-group-item:hover{color:#555;text-decoration:none;background-color:#f5f5f5} +button.list-group-item{width:100%;text-align:left} +.list-group-item-success{color:#3c763d;background-color:#dff0d8} +a.list-group-item-success,button.list-group-item-success{color:#3c763d} +a.list-group-item-success .list-group-item-heading,button.list-group-item-success .list-group-item-heading{color:inherit} +a.list-group-item-success:focus,a.list-group-item-success:hover,button.list-group-item-success:focus,button.list-group-item-success:hover{color:#3c763d;background-color:#d0e9c6} +a.list-group-item-success.active,a.list-group-item-success.active:focus,a.list-group-item-success.active:hover,button.list-group-item-success.active,button.list-group-item-success.active:focus,button.list-group-item-success.active:hover{color:#fff;background-color:#3c763d;border-color:#3c763d} +.list-group-item-info{color:#31708f;background-color:#d9edf7} +a.list-group-item-info,button.list-group-item-info{color:#31708f} +a.list-group-item-info .list-group-item-heading,button.list-group-item-info .list-group-item-heading{color:inherit} +a.list-group-item-info:focus,a.list-group-item-info:hover,button.list-group-item-info:focus,button.list-group-item-info:hover{color:#31708f;background-color:#c4e3f3} +a.list-group-item-info.active,a.list-group-item-info.active:focus,a.list-group-item-info.active:hover,button.list-group-item-info.active,button.list-group-item-info.active:focus,button.list-group-item-info.active:hover{color:#fff;background-color:#31708f;border-color:#31708f} +.list-group-item-warning{color:#8a6d3b;background-color:#fcf8e3} +a.list-group-item-warning,button.list-group-item-warning{color:#8a6d3b} +a.list-group-item-warning .list-group-item-heading,button.list-group-item-warning .list-group-item-heading{color:inherit} +a.list-group-item-warning:focus,a.list-group-item-warning:hover,button.list-group-item-warning:focus,button.list-group-item-warning:hover{color:#8a6d3b;background-color:#faf2cc} +a.list-group-item-warning.active,a.list-group-item-warning.active:focus,a.list-group-item-warning.active:hover,button.list-group-item-warning.active,button.list-group-item-warning.active:focus,button.list-group-item-warning.active:hover{color:#fff;background-color:#8a6d3b;border-color:#8a6d3b} +.list-group-item-danger{color:#a94442;background-color:#f2dede} +a.list-group-item-danger,button.list-group-item-danger{color:#a94442} +a.list-group-item-danger .list-group-item-heading,button.list-group-item-danger .list-group-item-heading{color:inherit} +a.list-group-item-danger:focus,a.list-group-item-danger:hover,button.list-group-item-danger:focus,button.list-group-item-danger:hover{color:#a94442;background-color:#ebcccc} +a.list-group-item-danger.active,a.list-group-item-danger.active:focus,a.list-group-item-danger.active:hover,button.list-group-item-danger.active,button.list-group-item-danger.active:focus,button.list-group-item-danger.active:hover{color:#fff;background-color:#a94442;border-color:#a94442} +.list-group-item-heading{margin-top:0;margin-bottom:5px} +.list-group-item-text{margin-bottom:0;line-height:1.3} +.panel{margin-bottom:20px;background-color:#fff;border:1px solid transparent;border-radius:4px;-webkit-box-shadow:0 1px 1px rgba(0,0,0,.05);box-shadow:0 1px 1px rgba(0,0,0,.05)} +.panel-body{padding:15px} +.panel-heading{padding:10px 15px;border-bottom:1px solid transparent;border-top-left-radius:3px;border-top-right-radius:3px} +.panel-heading>.dropdown .dropdown-toggle{color:inherit} +.panel-title{margin-top:0;margin-bottom:0;font-size:16px;color:inherit} +.panel-title>.small,.panel-title>.small>a,.panel-title>a,.panel-title>small,.panel-title>small>a{color:inherit} +.panel-footer{padding:10px 15px;background-color:#f5f5f5;border-top:1px solid #ddd;border-bottom-right-radius:3px;border-bottom-left-radius:3px} +.panel>.list-group,.panel>.panel-collapse>.list-group{margin-bottom:0} +.panel>.list-group .list-group-item,.panel>.panel-collapse>.list-group .list-group-item{border-width:1px 0;border-radius:0} +.panel>.list-group:first-child .list-group-item:first-child,.panel>.panel-collapse>.list-group:first-child .list-group-item:first-child{border-top:0;border-top-left-radius:3px;border-top-right-radius:3px} +.panel>.list-group:last-child .list-group-item:last-child,.panel>.panel-collapse>.list-group:last-child .list-group-item:last-child{border-bottom:0;border-bottom-right-radius:3px;border-bottom-left-radius:3px} +.panel>.panel-heading+.panel-collapse>.list-group .list-group-item:first-child{border-top-left-radius:0;border-top-right-radius:0} +.list-group+.panel-footer,.panel-heading+.list-group .list-group-item:first-child{border-top-width:0} +.panel>.panel-collapse>.table,.panel>.table,.panel>.table-responsive>.table{margin-bottom:0} +.panel>.panel-collapse>.table caption,.panel>.table caption,.panel>.table-responsive>.table caption{padding-right:15px;padding-left:15px} +.panel>.table-responsive:first-child>.table:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child,.panel>.table:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child,.panel>.table:first-child>thead:first-child>tr:first-child{border-top-left-radius:3px;border-top-right-radius:3px} +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:first-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:first-child,.panel>.table:first-child>thead:first-child>tr:first-child td:first-child,.panel>.table:first-child>thead:first-child>tr:first-child th:first-child{border-top-left-radius:3px} +.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table-responsive:first-child>.table:first-child>thead:first-child>tr:first-child th:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child td:last-child,.panel>.table:first-child>tbody:first-child>tr:first-child th:last-child,.panel>.table:first-child>thead:first-child>tr:first-child td:last-child,.panel>.table:first-child>thead:first-child>tr:first-child th:last-child{border-top-right-radius:3px} +.panel>.table-responsive:last-child>.table:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child,.panel>.table:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child{border-bottom-right-radius:3px;border-bottom-left-radius:3px} +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:first-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:first-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:first-child{border-bottom-left-radius:3px} +.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table-responsive:last-child>.table:last-child>tfoot:last-child>tr:last-child th:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child td:last-child,.panel>.table:last-child>tbody:last-child>tr:last-child th:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child td:last-child,.panel>.table:last-child>tfoot:last-child>tr:last-child th:last-child{border-bottom-right-radius:3px} +.panel>.panel-body+.table,.panel>.panel-body+.table-responsive,.panel>.table+.panel-body,.panel>.table-responsive+.panel-body{border-top:1px solid #ddd} +.panel>.table>tbody:first-child>tr:first-child td,.panel>.table>tbody:first-child>tr:first-child th{border-top:0} +.panel>.table-bordered,.panel>.table-responsive>.table-bordered{border:0} +.panel>.table-bordered>tbody>tr>td:first-child,.panel>.table-bordered>tbody>tr>th:first-child,.panel>.table-bordered>tfoot>tr>td:first-child,.panel>.table-bordered>tfoot>tr>th:first-child,.panel>.table-bordered>thead>tr>td:first-child,.panel>.table-bordered>thead>tr>th:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:first-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:first-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:first-child,.panel>.table-responsive>.table-bordered>thead>tr>td:first-child,.panel>.table-responsive>.table-bordered>thead>tr>th:first-child{border-left:0} +.panel>.table-bordered>tbody>tr>td:last-child,.panel>.table-bordered>tbody>tr>th:last-child,.panel>.table-bordered>tfoot>tr>td:last-child,.panel>.table-bordered>tfoot>tr>th:last-child,.panel>.table-bordered>thead>tr>td:last-child,.panel>.table-bordered>thead>tr>th:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>td:last-child,.panel>.table-responsive>.table-bordered>tbody>tr>th:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>td:last-child,.panel>.table-responsive>.table-bordered>tfoot>tr>th:last-child,.panel>.table-responsive>.table-bordered>thead>tr>td:last-child,.panel>.table-responsive>.table-bordered>thead>tr>th:last-child{border-right:0} +.panel>.table-bordered>tbody>tr:first-child>td,.panel>.table-bordered>tbody>tr:first-child>th,.panel>.table-bordered>tbody>tr:last-child>td,.panel>.table-bordered>tbody>tr:last-child>th,.panel>.table-bordered>tfoot>tr:last-child>td,.panel>.table-bordered>tfoot>tr:last-child>th,.panel>.table-bordered>thead>tr:first-child>td,.panel>.table-bordered>thead>tr:first-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:first-child>th,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>td,.panel>.table-responsive>.table-bordered>tbody>tr:last-child>th,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>td,.panel>.table-responsive>.table-bordered>tfoot>tr:last-child>th,.panel>.table-responsive>.table-bordered>thead>tr:first-child>td,.panel>.table-responsive>.table-bordered>thead>tr:first-child>th{border-bottom:0} +.panel>.table-responsive{margin-bottom:0;border:0} +.panel-group{margin-bottom:20px} +.panel-group .panel{margin-bottom:0;border-radius:4px} +.panel-group .panel+.panel{margin-top:5px} +.panel-group .panel-heading{border-bottom:0} +.panel-group .panel-heading+.panel-collapse>.list-group,.panel-group .panel-heading+.panel-collapse>.panel-body{border-top:1px solid #ddd} +.panel-group .panel-footer{border-top:0} +.panel-group .panel-footer+.panel-collapse .panel-body{border-bottom:1px solid #ddd} +.panel-default{border-color:#ddd} +.panel-default>.panel-heading{color:#333;background-color:#f5f5f5;border-color:#ddd} +.panel-default>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ddd} +.panel-default>.panel-heading .badge{color:#f5f5f5;background-color:#333} +.panel-default>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ddd} +.panel-primary{border-color:#337ab7} +.panel-primary>.panel-heading{color:#fff;background-color:#337ab7;border-color:#337ab7} +.panel-primary>.panel-heading+.panel-collapse>.panel-body{border-top-color:#337ab7} +.panel-primary>.panel-heading .badge{color:#337ab7;background-color:#fff} +.panel-primary>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#337ab7} +.panel-success{border-color:#d6e9c6} +.panel-success>.panel-heading{color:#3c763d;background-color:#dff0d8;border-color:#d6e9c6} +.panel-success>.panel-heading+.panel-collapse>.panel-body{border-top-color:#d6e9c6} +.panel-success>.panel-heading .badge{color:#dff0d8;background-color:#3c763d} +.panel-success>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#d6e9c6} +.panel-info{border-color:#bce8f1} +.panel-info>.panel-heading{color:#31708f;background-color:#d9edf7;border-color:#bce8f1} +.panel-info>.panel-heading+.panel-collapse>.panel-body{border-top-color:#bce8f1} +.panel-info>.panel-heading .badge{color:#d9edf7;background-color:#31708f} +.panel-info>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#bce8f1} +.panel-warning{border-color:#faebcc} +.panel-warning>.panel-heading{color:#8a6d3b;background-color:#fcf8e3;border-color:#faebcc} +.panel-warning>.panel-heading+.panel-collapse>.panel-body{border-top-color:#faebcc} +.panel-warning>.panel-heading .badge{color:#fcf8e3;background-color:#8a6d3b} +.panel-warning>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#faebcc} +.panel-danger{border-color:#ebccd1} +.panel-danger>.panel-heading{color:#a94442;background-color:#f2dede;border-color:#ebccd1} +.panel-danger>.panel-heading+.panel-collapse>.panel-body{border-top-color:#ebccd1} +.panel-danger>.panel-heading .badge{color:#f2dede;background-color:#a94442} +.panel-danger>.panel-footer+.panel-collapse>.panel-body{border-bottom-color:#ebccd1} +.embed-responsive{position:relative;display:block;height:0;padding:0;overflow:hidden} +.embed-responsive .embed-responsive-item,.embed-responsive embed,.embed-responsive iframe,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0} +.embed-responsive-16by9{padding-bottom:56.25%} +.embed-responsive-4by3{padding-bottom:75%} +.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,.05);box-shadow:inset 0 1px 1px rgba(0,0,0,.05)} +.well blockquote{border-color:rgba(0,0,0,.15)} +.well-lg{padding:24px;border-radius:6px} +.well-sm{padding:9px;border-radius:3px} +.close{float:right;font-size:21px;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.2} +.close:focus,.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.5} +button.close{padding:0;cursor:pointer;background:0 0;border:0;-webkit-appearance:none;-moz-appearance:none;appearance:none} +.modal-open{overflow:hidden} +.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;-webkit-overflow-scrolling:touch;outline:0} +.modal.fade .modal-dialog{-webkit-transform:translate(0,-25%);-ms-transform:translate(0,-25%);-o-transform:translate(0,-25%);transform:translate(0,-25%);-webkit-transition:-webkit-transform .3s ease-out;-o-transition:-o-transform .3s ease-out;transition:transform .3s ease-out;transition:transform .3s ease-out,-webkit-transform .3s ease-out,-o-transform .3s ease-out} +.modal.in .modal-dialog{-webkit-transform:translate(0,0);-ms-transform:translate(0,0);-o-transform:translate(0,0);transform:translate(0,0)} +.modal-open .modal{overflow-x:hidden;overflow-y:auto} +.modal-dialog{position:relative;width:auto;margin:10px} +.modal-content{position:relative;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:6px;-webkit-box-shadow:0 3px 9px rgba(0,0,0,.5);box-shadow:0 3px 9px rgba(0,0,0,.5);outline:0} +.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000} +.modal-backdrop.fade{opacity:0} +.modal-backdrop.in{opacity:.5} +.modal-header{padding:15px;border-bottom:1px solid #e5e5e5} +.modal-header .close{margin-top:-2px} +.modal-title{margin:0;line-height:1.42857143} +.modal-body{position:relative;padding:15px} +.modal-footer{padding:15px;text-align:right;border-top:1px solid #e5e5e5} +.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px} +.modal-footer .btn-group .btn+.btn{margin-left:-1px} +.modal-footer .btn-block+.btn-block{margin-left:0} +.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll} +@media (min-width:768px){ +.modal-dialog{width:600px;margin:30px auto} +.modal-content{-webkit-box-shadow:0 5px 15px rgba(0,0,0,.5);box-shadow:0 5px 15px rgba(0,0,0,.5)} +.modal-sm{width:300px} +} +@media (min-width:992px){ +.modal-lg{width:900px} +} +.tooltip{position:absolute;z-index:1070;display:block;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-style:normal;font-weight:400;line-height:1.42857143;line-break:auto;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;font-size:12px;opacity:0} +.tooltip.in{opacity:.9} +.tooltip.top{padding:5px 0;margin-top:-3px} +.tooltip.right{padding:0 5px;margin-left:3px} +.tooltip.bottom{padding:5px 0;margin-top:3px} +.tooltip.left{padding:0 5px;margin-left:-3px} +.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-width:5px 5px 0;border-top-color:#000} +.tooltip.top-left .tooltip-arrow{right:5px;bottom:0;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000} +.tooltip.top-right .tooltip-arrow{bottom:0;left:5px;margin-bottom:-5px;border-width:5px 5px 0;border-top-color:#000} +.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-width:5px 5px 5px 0;border-right-color:#000} +.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-width:5px 0 5px 5px;border-left-color:#000} +.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-width:0 5px 5px;border-bottom-color:#000} +.tooltip.bottom-left .tooltip-arrow{top:0;right:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000} +.tooltip.bottom-right .tooltip-arrow{top:0;left:5px;margin-top:-5px;border-width:0 5px 5px;border-bottom-color:#000} +.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;background-color:#000;border-radius:4px} +.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid} +.popover{position:absolute;top:0;left:0;z-index:1060;display:none;max-width:276px;padding:1px;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-style:normal;font-weight:400;line-height:1.42857143;line-break:auto;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;word-wrap:normal;white-space:normal;font-size:14px;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,.2);box-shadow:0 5px 10px rgba(0,0,0,.2)} +.popover.top{margin-top:-10px} +.popover.right{margin-left:10px} +.popover.bottom{margin-top:10px} +.popover.left{margin-left:-10px} +.popover>.arrow{border-width:11px} +.popover>.arrow,.popover>.arrow:after{position:absolute;display:block;width:0;height:0;border-color:transparent;border-style:solid} +.popover>.arrow:after{content:"";border-width:10px} +.popover.top>.arrow{bottom:-11px;left:50%;margin-left:-11px;border-top-color:rgba(0,0,0,.25);border-bottom-width:0} +.popover.top>.arrow:after{bottom:1px;margin-left:-10px;content:" ";border-top-color:#fff;border-bottom-width:0} +.popover.right>.arrow{top:50%;left:-11px;margin-top:-11px;border-right-color:rgba(0,0,0,.25);border-left-width:0} +.popover.right>.arrow:after{bottom:-10px;left:1px;content:" ";border-right-color:#fff;border-left-width:0} +.popover.bottom>.arrow{top:-11px;left:50%;margin-left:-11px;border-top-width:0;border-bottom-color:rgba(0,0,0,.25)} +.popover.bottom>.arrow:after{top:1px;margin-left:-10px;content:" ";border-top-width:0;border-bottom-color:#fff} +.popover.left>.arrow{top:50%;right:-11px;margin-top:-11px;border-right-width:0;border-left-color:rgba(0,0,0,.25)} +.popover.left>.arrow:after{right:1px;bottom:-10px;content:" ";border-right-width:0;border-left-color:#fff} +.popover-title{padding:8px 14px;margin:0;font-size:14px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-radius:5px 5px 0 0} +.popover-content{padding:9px 14px} +.carousel{position:relative} +.carousel-inner{position:relative;width:100%;overflow:hidden} +.carousel-inner>.item{position:relative;display:none;-webkit-transition:left .6s ease-in-out;-o-transition:left .6s ease-in-out;transition:left .6s ease-in-out} +.carousel-inner>.item>a>img,.carousel-inner>.item>img{line-height:1} +@media all and (transform-3d),(-webkit-transform-3d){ +.carousel-inner>.item{-webkit-transition:-webkit-transform .6s ease-in-out;-o-transition:-o-transform .6s ease-in-out;transition:transform .6s ease-in-out;transition:transform .6s ease-in-out,-webkit-transform .6s ease-in-out,-o-transform .6s ease-in-out;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px} +.carousel-inner>.item.active.right,.carousel-inner>.item.next{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0);left:0} +.carousel-inner>.item.active.left,.carousel-inner>.item.prev{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0);left:0} +.carousel-inner>.item.active,.carousel-inner>.item.next.left,.carousel-inner>.item.prev.right{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0);left:0} +} +.carousel-inner>.active,.carousel-inner>.next,.carousel-inner>.prev{display:block} +.carousel-inner>.active{left:0} +.carousel-inner>.next,.carousel-inner>.prev{position:absolute;top:0;width:100%} +.carousel-inner>.next{left:100%} +.carousel-inner>.prev{left:-100%} +.carousel-inner>.next.left,.carousel-inner>.prev.right{left:0} +.carousel-inner>.active.left{left:-100%} +.carousel-inner>.active.right{left:100%} +.carousel-control{position:absolute;top:0;bottom:0;left:0;width:15%;font-size:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6);background-color:rgba(0,0,0,0);opacity:.5} +.carousel-control.left{background-image:-webkit-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.5)),to(rgba(0,0,0,.0001)));background-image:linear-gradient(to right,rgba(0,0,0,.5) 0,rgba(0,0,0,.0001) 100%);background-repeat:repeat-x} +.carousel-control.right{right:0;left:auto;background-image:-webkit-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-o-linear-gradient(left,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.0001)),to(rgba(0,0,0,.5)));background-image:linear-gradient(to right,rgba(0,0,0,.0001) 0,rgba(0,0,0,.5) 100%);background-repeat:repeat-x} +.carousel-control:focus,.carousel-control:hover{color:#fff;text-decoration:none;outline:0;opacity:.9} +.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{position:absolute;top:50%;z-index:5;display:inline-block;margin-top:-10px} +.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{left:50%;margin-left:-10px} +.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{right:50%;margin-right:-10px} +.carousel-control .icon-next,.carousel-control .icon-prev{width:20px;height:20px;font-family:serif;line-height:1} +.carousel-control .icon-prev:before{content:"\2039"} +.carousel-control .icon-next:before{content:"\203a"} +.carousel-indicators{position:absolute;bottom:10px;left:50%;z-index:15;width:60%;padding-left:0;margin-left:-30%;text-align:center;list-style:none} +.carousel-indicators li{display:inline-block;width:10px;height:10px;margin:1px;text-indent:-999px;cursor:pointer;background-color:rgba(0,0,0,0);border:1px solid #fff;border-radius:10px} +.carousel-indicators .active{width:12px;height:12px;margin:0;background-color:#fff} +.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center;text-shadow:0 1px 2px rgba(0,0,0,.6)} +.carousel-caption .btn{text-shadow:none} +@media screen and (min-width:768px){ +.carousel-control .glyphicon-chevron-left,.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next,.carousel-control .icon-prev{width:30px;height:30px;margin-top:-10px;font-size:30px} +.carousel-control .glyphicon-chevron-left,.carousel-control .icon-prev{margin-left:-10px} +.carousel-control .glyphicon-chevron-right,.carousel-control .icon-next{margin-right:-10px} +.carousel-caption{right:20%;left:20%;padding-bottom:30px} +.carousel-indicators{bottom:20px} +} +.btn-group-vertical>.btn-group:after,.btn-group-vertical>.btn-group:before,.btn-toolbar:after,.btn-toolbar:before,.clearfix:after,.clearfix:before,.container-fluid:after,.container-fluid:before,.container:after,.container:before,.dl-horizontal dd:after,.dl-horizontal dd:before,.form-horizontal .form-group:after,.form-horizontal .form-group:before,.modal-footer:after,.modal-footer:before,.modal-header:after,.modal-header:before,.nav:after,.nav:before,.navbar-collapse:after,.navbar-collapse:before,.navbar-header:after,.navbar-header:before,.navbar:after,.navbar:before,.pager:after,.pager:before,.panel-body:after,.panel-body:before,.row:after,.row:before{display:table;content:" "} +.btn-group-vertical>.btn-group:after,.btn-toolbar:after,.clearfix:after,.container-fluid:after,.container:after,.dl-horizontal dd:after,.form-horizontal .form-group:after,.modal-footer:after,.modal-header:after,.nav:after,.navbar-collapse:after,.navbar-header:after,.navbar:after,.pager:after,.panel-body:after,.row:after{clear:both} +.center-block{display:block;margin-right:auto;margin-left:auto} +.pull-right{float:right!important} +.pull-left{float:left!important} +.hide{display:none!important} +.show{display:block!important} +.invisible{visibility:hidden} +.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0} +.hidden{display:none!important} +.affix{position:fixed} +@-ms-viewport{width:device-width} +.visible-lg,.visible-lg-block,.visible-lg-inline,.visible-lg-inline-block,.visible-md,.visible-md-block,.visible-md-inline,.visible-md-inline-block,.visible-sm,.visible-sm-block,.visible-sm-inline,.visible-sm-inline-block,.visible-xs,.visible-xs-block,.visible-xs-inline,.visible-xs-inline-block{display:none!important} +@media (max-width:767px){ +.visible-xs{display:block!important} +table.visible-xs{display:table!important} +tr.visible-xs{display:table-row!important} +td.visible-xs,th.visible-xs{display:table-cell!important} +.visible-xs-block{display:block!important} +.visible-xs-inline{display:inline!important} +.visible-xs-inline-block{display:inline-block!important} +} +@media (min-width:768px) and (max-width:991px){ +.visible-sm{display:block!important} +table.visible-sm{display:table!important} +tr.visible-sm{display:table-row!important} +td.visible-sm,th.visible-sm{display:table-cell!important} +.visible-sm-block{display:block!important} +.visible-sm-inline{display:inline!important} +.visible-sm-inline-block{display:inline-block!important} +} +@media (min-width:992px) and (max-width:1199px){ +.visible-md{display:block!important} +table.visible-md{display:table!important} +tr.visible-md{display:table-row!important} +td.visible-md,th.visible-md{display:table-cell!important} +.visible-md-block{display:block!important} +.visible-md-inline{display:inline!important} +.visible-md-inline-block{display:inline-block!important} +} +@media (min-width:1200px){ +.visible-lg{display:block!important} +table.visible-lg{display:table!important} +tr.visible-lg{display:table-row!important} +td.visible-lg,th.visible-lg{display:table-cell!important} +.visible-lg-block{display:block!important} +.visible-lg-inline{display:inline!important} +.visible-lg-inline-block{display:inline-block!important} +.hidden-lg{display:none!important} +} +@media (max-width:767px){ +.hidden-xs{display:none!important} +} +@media (min-width:768px) and (max-width:991px){ +.hidden-sm{display:none!important} +} +@media (min-width:992px) and (max-width:1199px){ +.hidden-md{display:none!important} +} +.visible-print{display:none!important} +@media print{ +.visible-print{display:block!important} +table.visible-print{display:table!important} +tr.visible-print{display:table-row!important} +td.visible-print,th.visible-print{display:table-cell!important} +} +.visible-print-block{display:none!important} +@media print{ +.visible-print-block{display:block!important} +} +.visible-print-inline{display:none!important} +@media print{ +.visible-print-inline{display:inline!important} +} +.visible-print-inline-block{display:none!important} +@media print{ +.visible-print-inline-block{display:inline-block!important} +.hidden-print{display:none!important} +} +.hljs{display:block;background:#fff;padding:.5em;color:#333;overflow-x:auto} +.hljs-comment,.hljs-meta{color:#969896} +.hljs-emphasis,.hljs-quote,.hljs-string,.hljs-strong,.hljs-template-variable,.hljs-variable{color:#df5000} +.hljs-keyword,.hljs-selector-tag,.hljs-type{color:#a71d5d} +.hljs-attribute,.hljs-bullet,.hljs-literal,.hljs-symbol{color:#0086b3} +.hljs-name,.hljs-section{color:#63a35c} +.hljs-tag{color:#333} +.hljs-attr,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-selector-pseudo,.hljs-title{color:#795da3} +.hljs-addition{color:#55a532;background-color:#eaffea} +.hljs-deletion{color:#bd2c00;background-color:#ffecec} +.hljs-link{text-decoration:underline} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/styles/docfx.vendor.js b/csharp/ApiDocs/_exported_templates/default/styles/docfx.vendor.js new file mode 100644 index 0000000000..e565782944 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/docfx.vendor.js @@ -0,0 +1 @@ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,(function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.5.1",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"�":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be((function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()}),{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le((function(o){return o=+o,le((function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))}))}))}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e.namespaceURI,n=(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce((function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length})),d.attributes=ce((function(e){return e.className="i",!e.getAttribute("className")})),d.getElementsByTagName=ce((function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length})),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce((function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length})),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce((function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")})),ce((function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")}))),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce((function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)})),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},D=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,(function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")}))},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function D(e,n,r){return m(n)?S.grep(e,(function(e,t){return!!n.call(e,t,e)!==r})):n.nodeType?S.grep(e,(function(e){return e===n!==r})):"string"!=typeof n?S.grep(e,(function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||j,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,j=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter((function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
      "],col:[2,"","
      "],tr:[2,"","
      "],td:[3,"","
      "],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}}));var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",(function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always((function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0})),"script"})),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="
      ",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)})).always(n&&function(e,t){a.each((function(){n.apply(this,o||[e.responseText,t,e])}))}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,(function(e){return t===e.elem})).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each((function(e){S.offset.setOffset(this,t,e)}));var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map((function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re}))}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},(function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,(function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n}),t,e,arguments.length)}})),S.each(["top","left"],(function(e,n){S.cssHooks[n]=$e(y.pixelPosition,(function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t}))})),S.each({Height:"height",Width:"width"},(function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},(function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,(function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)}),s,n?e:void 0,n)}}))})),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],(function(e,t){S.fn[t]=function(e){return this.on(t,e)}})),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),(function(e,n){S.fn[n]=function(e,t){return 0this.$items.length-1||t<0))return this.sliding?this.$element.one("slid.bs.carousel",(function(){e.to(t)})):i==t?this.pause().cycle():this.slide(idocument.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&t?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!t?this.scrollbarWidth:""})},s.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},s.prototype.checkScrollbar=function(){var t=window.innerWidth;if(!t){var e=document.documentElement.getBoundingClientRect();t=e.right-Math.abs(e.left)}this.bodyIsOverflowing=document.body.clientWidth
      ',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0},sanitize:!0,sanitizeFn:null,whiteList:t},m.prototype.init=function(t,e,i){if(this.enabled=!0,this.type=t,this.$element=g(e),this.options=this.getOptions(i),this.$viewport=this.options.viewport&&g(document).find(g.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var o=this.options.trigger.split(" "),n=o.length;n--;){var s=o[n];if("click"==s)this.$element.on("click."+this.type,this.options.selector,g.proxy(this.toggle,this));else if("manual"!=s){var a="hover"==s?"mouseenter":"focusin",r="hover"==s?"mouseleave":"focusout";this.$element.on(a+"."+this.type,this.options.selector,g.proxy(this.enter,this)),this.$element.on(r+"."+this.type,this.options.selector,g.proxy(this.leave,this))}}this.options.selector?this._options=g.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},m.prototype.getDefaults=function(){return m.DEFAULTS},m.prototype.getOptions=function(t){var e=this.$element.data();for(var i in e)e.hasOwnProperty(i)&&-1!==g.inArray(i,o)&&delete e[i];return(t=g.extend({},this.getDefaults(),e,t)).delay&&"number"==typeof t.delay&&(t.delay={show:t.delay,hide:t.delay}),t.sanitize&&(t.template=n(t.template,t.whiteList,t.sanitizeFn)),t},m.prototype.getDelegateOptions=function(){var i={},o=this.getDefaults();return this._options&&g.each(this._options,(function(t,e){o[t]!=e&&(i[t]=e)})),i},m.prototype.enter=function(t){var e=t instanceof this.constructor?t:g(t.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e)),t instanceof g.Event&&(e.inState["focusin"==t.type?"focus":"hover"]=!0),e.tip().hasClass("in")||"in"==e.hoverState)e.hoverState="in";else{if(clearTimeout(e.timeout),e.hoverState="in",!e.options.delay||!e.options.delay.show)return e.show();e.timeout=setTimeout((function(){"in"==e.hoverState&&e.show()}),e.options.delay.show)}},m.prototype.isInStateTrue=function(){for(var t in this.inState)if(this.inState[t])return!0;return!1},m.prototype.leave=function(t){var e=t instanceof this.constructor?t:g(t.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e)),t instanceof g.Event&&(e.inState["focusout"==t.type?"focus":"hover"]=!1),!e.isInStateTrue()){if(clearTimeout(e.timeout),e.hoverState="out",!e.options.delay||!e.options.delay.hide)return e.hide();e.timeout=setTimeout((function(){"out"==e.hoverState&&e.hide()}),e.options.delay.hide)}},m.prototype.show=function(){var t=g.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(t);var e=g.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(t.isDefaultPrevented()||!e)return;var i=this,o=this.tip(),n=this.getUID(this.type);this.setContent(),o.attr("id",n),this.$element.attr("aria-describedby",n),this.options.animation&&o.addClass("fade");var s="function"==typeof this.options.placement?this.options.placement.call(this,o[0],this.$element[0]):this.options.placement,a=/\s?auto?\s?/i,r=a.test(s);r&&(s=s.replace(a,"")||"top"),o.detach().css({top:0,left:0,display:"block"}).addClass(s).data("bs."+this.type,this),this.options.container?o.appendTo(g(document).find(this.options.container)):o.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var l=this.getPosition(),h=o[0].offsetWidth,d=o[0].offsetHeight;if(r){var p=s,c=this.getPosition(this.$viewport);s="bottom"==s&&l.bottom+d>c.bottom?"top":"top"==s&&l.top-dc.width?"left":"left"==s&&l.left-ha.top+a.height&&(n.top=a.top+a.height-l)}else{var h=e.left-s,d=e.left+s+i;ha.right&&(n.left=a.left+a.width-d)}return n},m.prototype.getTitle=function(){var t=this.$element,e=this.options;return t.attr("data-original-title")||("function"==typeof e.title?e.title.call(t[0]):e.title)},m.prototype.getUID=function(t){for(;t+=~~(1e6*Math.random()),document.getElementById(t););return t},m.prototype.tip=function(){if(!this.$tip&&(this.$tip=g(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},m.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},m.prototype.enable=function(){this.enabled=!0},m.prototype.disable=function(){this.enabled=!1},m.prototype.toggleEnabled=function(){this.enabled=!this.enabled},m.prototype.toggle=function(t){var e=this;t&&((e=g(t.currentTarget).data("bs."+this.type))||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e))),t?(e.inState.click=!e.inState.click,e.isInStateTrue()?e.enter(e):e.leave(e)):e.tip().hasClass("in")?e.leave(e):e.enter(e)},m.prototype.destroy=function(){var t=this;clearTimeout(this.timeout),this.hide((function(){t.$element.off("."+t.type).removeData("bs."+t.type),t.$tip&&t.$tip.detach(),t.$tip=null,t.$arrow=null,t.$viewport=null,t.$element=null}))},m.prototype.sanitizeHtml=function(t){return n(t,this.options.whiteList,this.options.sanitizeFn)};var e=g.fn.tooltip;g.fn.tooltip=function i(o){return this.each((function(){var t=g(this),e=t.data("bs.tooltip"),i="object"==typeof o&&o;!e&&/destroy|hide/.test(o)||(e||t.data("bs.tooltip",e=new m(this,i)),"string"==typeof o&&e[o]())}))},g.fn.tooltip.Constructor=m,g.fn.tooltip.noConflict=function(){return g.fn.tooltip=e,this}}(jQuery),function(n){"use strict";var s=function(t,e){this.init("popover",t,e)};if(!n.fn.tooltip)throw new Error("Popover requires tooltip.js");s.VERSION="3.4.1",s.DEFAULTS=n.extend({},n.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:''}),((s.prototype=n.extend({},n.fn.tooltip.Constructor.prototype)).constructor=s).prototype.getDefaults=function(){return s.DEFAULTS},s.prototype.setContent=function(){var t=this.tip(),e=this.getTitle(),i=this.getContent();if(this.options.html){var o=typeof i;this.options.sanitize&&(e=this.sanitizeHtml(e),"string"===o&&(i=this.sanitizeHtml(i))),t.find(".popover-title").html(e),t.find(".popover-content").children().detach().end()["string"===o?"html":"append"](i)}else t.find(".popover-title").text(e),t.find(".popover-content").children().detach().end().text(i);t.removeClass("fade top bottom left right in"),t.find(".popover-title").html()||t.find(".popover-title").hide()},s.prototype.hasContent=function(){return this.getTitle()||this.getContent()},s.prototype.getContent=function(){var t=this.$element,e=this.options;return t.attr("data-content")||("function"==typeof e.content?e.content.call(t[0]):e.content)},s.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var t=n.fn.popover;n.fn.popover=function e(o){return this.each((function(){var t=n(this),e=t.data("bs.popover"),i="object"==typeof o&&o;!e&&/destroy|hide/.test(o)||(e||t.data("bs.popover",e=new s(this,i)),"string"==typeof o&&e[o]())}))},n.fn.popover.Constructor=s,n.fn.popover.noConflict=function(){return n.fn.popover=t,this}}(jQuery),function(s){"use strict";function n(t,e){this.$body=s(document.body),this.$scrollElement=s(t).is(document.body)?s(window):s(t),this.options=s.extend({},n.DEFAULTS,e),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",s.proxy(this.process,this)),this.refresh(),this.process()}function e(o){return this.each((function(){var t=s(this),e=t.data("bs.scrollspy"),i="object"==typeof o&&o;e||t.data("bs.scrollspy",e=new n(this,i)),"string"==typeof o&&e[o]()}))}n.VERSION="3.4.1",n.DEFAULTS={offset:10},n.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},n.prototype.refresh=function(){var t=this,o="offset",n=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),s.isWindow(this.$scrollElement[0])||(o="position",n=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map((function(){var t=s(this),e=t.data("target")||t.attr("href"),i=/^#./.test(e)&&s(e);return i&&i.length&&i.is(":visible")&&[[i[o]().top+n,e]]||null})).sort((function(t,e){return t[0]-e[0]})).each((function(){t.offsets.push(this[0]),t.targets.push(this[1])}))},n.prototype.process=function(){var t,e=this.$scrollElement.scrollTop()+this.options.offset,i=this.getScrollHeight(),o=this.options.offset+i-this.$scrollElement.height(),n=this.offsets,s=this.targets,a=this.activeTarget;if(this.scrollHeight!=i&&this.refresh(),o<=e)return a!=(t=s[s.length-1])&&this.activate(t);if(a&&e=n[t]&&(n[t+1]===undefined||e .active"),n=i&&r.support.transition&&(o.length&&o.hasClass("fade")||!!e.find("> .fade").length);function s(){o.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),t.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),n?(t[0].offsetWidth,t.addClass("in")):t.removeClass("fade"),t.parent(".dropdown-menu").length&&t.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),i&&i()}o.length&&n?o.one("bsTransitionEnd",s).emulateTransitionEnd(a.TRANSITION_DURATION):s(),o.removeClass("in")};var t=r.fn.tab;r.fn.tab=e,r.fn.tab.Constructor=a,r.fn.tab.noConflict=function(){return r.fn.tab=t,this};var i=function(t){t.preventDefault(),e.call(r(this),"show")};r(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',i).on("click.bs.tab.data-api",'[data-toggle="pill"]',i)}(jQuery),function(l){"use strict";var h=function(t,e){this.options=l.extend({},h.DEFAULTS,e);var i=this.options.target===h.DEFAULTS.target?l(this.options.target):l(document).find(this.options.target);this.$target=i.on("scroll.bs.affix.data-api",l.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",l.proxy(this.checkPositionWithEventLoop,this)),this.$element=l(t),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};function i(o){return this.each((function(){var t=l(this),e=t.data("bs.affix"),i="object"==typeof o&&o;e||t.data("bs.affix",e=new h(this,i)),"string"==typeof o&&e[o]()}))}h.VERSION="3.4.1",h.RESET="affix affix-top affix-bottom",h.DEFAULTS={offset:0,target:window},h.prototype.getState=function(t,e,i,o){var n=this.$target.scrollTop(),s=this.$element.offset(),a=this.$target.height();if(null!=i&&"top"==this.affixed)return n/g,">")}function r(e){return e.nodeName.toLowerCase()}function a(e,t){var r=e&&e.exec(t);return r&&0===r.index}function i(e){return T.test(e)}function n(e){var t,r,a,n,o=e.className+" ";if(o+=e.parentNode?e.parentNode.className:"",r=w.exec(o))return S(r[1])?r[1]:"no-highlight";for(o=o.split(/\s+/),t=0,a=o.length;a>t;t++)if(n=o[t],i(n)||S(n))return n}function o(e){var t,r={},a=Array.prototype.slice.call(arguments,1);for(t in e)r[t]=e[t];return a.forEach((function(e){for(t in e)r[t]=e[t]})),r}function s(e){var t=[];return function a(e,i){for(var n=e.firstChild;n;n=n.nextSibling)3===n.nodeType?i+=n.nodeValue.length:1===n.nodeType&&(t.push({event:"start",offset:i,node:n}),i=a(n,i),r(n).match(/br|hr|img|input/)||t.push({event:"stop",offset:i,node:n}));return i}(e,0),t}function l(e,a,i){function n(){return e.length&&a.length?e[0].offset!==a[0].offset?e[0].offset"}function s(e){d+=""}function l(e){("start"===e.event?o:s)(e.node)}for(var c=0,d="",p=[];e.length||a.length;){var m=n();if(d+=t(i.substring(c,m[0].offset)),c=m[0].offset,m===e){p.reverse().forEach(s);do{l(m.splice(0,1)[0]),m=n()}while(m===e&&m.length&&m[0].offset===c);p.reverse().forEach(o)}else"start"===m[0].event?p.push(m[0].node):p.pop(),l(m.splice(0,1)[0])}return d+t(i.substr(c))}function c(e){return e.v&&!e.cached_variants&&(e.cached_variants=e.v.map((function(t){return o(e,{v:null},t)}))),e.cached_variants||e.eW&&[o(e)]||[e]}function d(e){function t(e){return e&&e.source||e}function r(r,a){return new RegExp(t(r),"m"+(e.cI?"i":"")+(a?"g":""))}function a(i,n){if(!i.compiled){if(i.compiled=!0,i.k=i.k||i.bK,i.k){var o={},s=function(t,r){e.cI&&(r=r.toLowerCase()),r.split(" ").forEach((function(e){var r=e.split("|");o[r[0]]=[t,r[1]?Number(r[1]):1]}))};"string"==typeof i.k?s("keyword",i.k):x(i.k).forEach((function(e){s(e,i.k[e])})),i.k=o}i.lR=r(i.l||/\w+/,!0),n&&(i.bK&&(i.b="\\b("+i.bK.split(" ").join("|")+")\\b"),i.b||(i.b=/\B|\b/),i.bR=r(i.b),i.e||i.eW||(i.e=/\B|\b/),i.e&&(i.eR=r(i.e)),i.tE=t(i.e)||"",i.eW&&n.tE&&(i.tE+=(i.e?"|":"")+n.tE)),i.i&&(i.iR=r(i.i)),null==i.r&&(i.r=1),i.c||(i.c=[]),i.c=Array.prototype.concat.apply([],i.c.map((function(e){return c("self"===e?i:e)}))),i.c.forEach((function(e){a(e,i)})),i.starts&&a(i.starts,n);var l=i.c.map((function(e){return e.bK?"\\.?("+e.b+")\\.?":e.b})).concat([i.tE,i.i]).map(t).filter(Boolean);i.t=l.length?r(l.join("|"),!0):{exec:function(){return null}}}}a(e)}function p(e,r,i,n){function o(e,t){var r,i;for(r=0,i=t.c.length;i>r;r++)if(a(t.c[r].bR,e))return t.c[r]}function s(e,t){if(a(e.eR,t)){for(;e.endsParent&&e.parent;)e=e.parent;return e}return e.eW?s(e.parent,t):void 0}function l(e,t){return!i&&a(t.iR,e)}function c(e,t){var r=v.cI?t[0].toLowerCase():t[0];return e.k.hasOwnProperty(r)&&e.k[r]}function u(e,t,r,a){var i=a?"":D.classPrefix,n='',n+t+o}function b(){var e,r,a,i;if(!C.k)return t(T);for(i="",r=0,C.lR.lastIndex=0,a=C.lR.exec(T);a;)i+=t(T.substring(r,a.index)),e=c(C,a),e?(w+=e[1],i+=u(e[0],t(a[0]))):i+=t(a[0]),r=C.lR.lastIndex,a=C.lR.exec(T);return i+t(T.substr(r))}function g(){var e="string"==typeof C.sL;if(e&&!E[C.sL])return t(T);var r=e?p(C.sL,T,!0,x[C.sL]):m(T,C.sL.length?C.sL:void 0);return C.r>0&&(w+=r.r),e&&(x[C.sL]=r.top),u(r.language,r.value,!1,!0)}function f(){N+=null!=C.sL?g():b(),T=""}function _(e){N+=e.cN?u(e.cN,"",!0):"",C=Object.create(e,{parent:{value:C}})}function h(e,t){if(T+=e,null==t)return f(),0;var r=o(t,C);if(r)return r.skip?T+=t:(r.eB&&(T+=t),f(),r.rB||r.eB||(T=t)),_(r,t),r.rB?0:t.length;var a=s(C,t);if(a){var i=C;i.skip?T+=t:(i.rE||i.eE||(T+=t),f(),i.eE&&(T=t));do{C.cN&&(N+=M),C.skip||(w+=C.r),C=C.parent}while(C!==a.parent);return a.starts&&_(a.starts,""),i.rE?0:t.length}if(l(t,C))throw new Error('Illegal lexeme "'+t+'" for mode "'+(C.cN||"")+'"');return T+=t,t.length||1}var v=S(e);if(!v)throw new Error('Unknown language: "'+e+'"');d(v);var y,C=n||v,x={},N="";for(y=C;y!==v;y=y.parent)y.cN&&(N=u(y.cN,"",!0)+N);var T="",w=0;try{for(var A,I,k=0;;){if(C.t.lastIndex=k,A=C.t.exec(r),!A)break;I=h(r.substring(k,A.index),A[0]),k=A.index+I}for(h(r.substr(k)),y=C;y.parent;y=y.parent)y.cN&&(N+=M);return{r:w,value:N,language:e,top:C}}catch(R){if(R.message&&-1!==R.message.indexOf("Illegal"))return{r:0,value:t(r)};throw R}}function m(e,r){r=r||D.languages||x(E);var a={r:0,value:t(e)},i=a;return r.filter(S).forEach((function(t){var r=p(t,e,!1);r.language=t,r.r>i.r&&(i=r),r.r>a.r&&(i=a,a=r)})),i.language&&(a.second_best=i),a}function u(e){return D.tabReplace||D.useBR?e.replace(A,(function(e,t){return D.useBR&&"\n"===e?"
      ":D.tabReplace?t.replace(/\t/g,D.tabReplace):""})):e}function b(e,t,r){var a=t?N[t]:r,i=[e.trim()];return e.match(/\bhljs\b/)||i.push("hljs"),-1===e.indexOf(a)&&i.push(a),i.join(" ").trim()}function g(e){var t,r,a,o,c,d=n(e);i(d)||(D.useBR?(t=document.createElementNS("http://www.w3.org/1999/xhtml","div"),t.innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n")):t=e,c=t.textContent,a=d?p(d,c,!0):m(c),r=s(t),r.length&&(o=document.createElementNS("http://www.w3.org/1999/xhtml","div"),o.innerHTML=a.value,a.value=l(r,s(o),c)),a.value=u(a.value),e.innerHTML=a.value,e.className=b(e.className,d,a.language),e.result={language:a.language,re:a.r},a.second_best&&(e.second_best={language:a.second_best.language,re:a.second_best.r}))}function f(e){D=o(D,e)}function _(){if(!_.called){_.called=!0;var e=document.querySelectorAll("pre code");C.forEach.call(e,g)}}function h(){addEventListener("DOMContentLoaded",_,!1),addEventListener("load",_,!1)}function v(t,r){var a=E[t]=r(e);a.aliases&&a.aliases.forEach((function(e){N[e]=t}))}function y(){return x(E)}function S(e){return e=(e||"").toLowerCase(),E[e]||E[N[e]]}var C=[],x=Object.keys,E={},N={},T=/^(no-?highlight|plain|text)$/i,w=/\blang(?:uage)?-([\w-]+)\b/i,A=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,M="      ",D={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0};return e.highlight=p,e.highlightAuto=m,e.fixMarkup=u,e.highlightBlock=g,e.configure=f,e.initHighlighting=_,e.initHighlightingOnLoad=h,e.registerLanguage=v,e.listLanguages=y,e.getLanguage=S,e.inherit=o,e.IR="[a-zA-Z]\\w*",e.UIR="[a-zA-Z_]\\w*",e.NR="\\b\\d+(\\.\\d+)?",e.CNR="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",e.BNR="\\b(0b[01]+)",e.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",e.BE={b:"\\\\[\\s\\S]",r:0},e.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[e.BE]},e.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[e.BE]},e.PWM={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},e.C=function(t,r,a){var i=e.inherit({cN:"comment",b:t,e:r,c:[]},a||{});return i.c.push(e.PWM),i.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",r:0}),i},e.CLCM=e.C("//","$"),e.CBCM=e.C("/\\*","\\*/"),e.HCM=e.C("#","$"),e.NM={cN:"number",b:e.NR,r:0},e.CNM={cN:"number",b:e.CNR,r:0},e.BNM={cN:"number",b:e.BNR,r:0},e.CSSNM={cN:"number",b:e.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",r:0},e.RM={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[e.BE,{b:/\[/,e:/\]/,r:0,c:[e.BE]}]},e.TM={cN:"title",b:e.IR,r:0},e.UTM={cN:"title",b:e.UIR,r:0},e.METHOD_GUARD={b:"\\.\\s*"+e.UIR,r:0},e.registerLanguage("1c",(function(e){var t="[A-Za-zÐ-Яа-ÑÑ‘Ð_][A-Za-zÐ-Яа-ÑÑ‘Ð_0-9]+",r="далее ",a="возврат вызватьиÑключение выполнить Ð´Ð»Ñ ÐµÑли и из или иначе иначееÑли иÑключение каждого конецеÑли конецпопытки конеццикла не новый перейти перем по пока попытка прервать продолжить тогда цикл ÑкÑпорт ",i=r+a,n="загрузитьизфайла ",o="вебклиент вмеÑто внешнееÑоединение клиент конецоблаÑти мобильноеприложениеклиент мобильноеприложениеÑервер наклиенте наклиентенаÑервере наклиентенаÑерверебезконтекÑта наÑервере наÑерверебезконтекÑта облаÑÑ‚ÑŒ перед поÑле Ñервер толÑтыйклиентобычноеприложение толÑтыйклиентуправлÑемоеприложение тонкийклиент ",s=n+o,l="разделительÑтраниц разделительÑтрок ÑимволтабулÑции ",c="ansitooem oemtoansi ввеÑтивидÑубконто ввеÑтиперечиÑление ввеÑтипериод ввеÑтипланÑчетов выбранныйпланÑчетов датагод датамеÑÑц датачиÑло заголовокÑиÑтемы значениевÑтроку значениеизÑтроки каталогиб ÐºÐ°Ñ‚Ð°Ð»Ð¾Ð³Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ ÐºÐ¾Ð´Ñимв конгода конецпериодаби конецраÑÑчитанногопериодаби конецÑтандартногоинтервала конквартала конмеÑÑца коннедели лог лог10 макÑимальноеколичеÑтвоÑубконто названиеинтерфейÑа названиенабораправ назначитьвид назначитьÑчет найтиÑÑылки началопериодаби началоÑтандартногоинтервала начгода начквартала начмеÑÑца начнедели номерднÑгода номерднÑнедели номернеделигода Ð¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚ÐºÐ°Ð¾Ð¶Ð¸Ð´Ð°Ð½Ð¸Ñ Ð¾ÑновнойжурналраÑчетов оÑновнойпланÑчетов оÑновнойÑзык очиÑтитьокноÑообщений периодÑÑ‚Ñ€ получитьвремÑта получитьдатута получитьдокументта получитьзначениÑотбора получитьпозициюта получитьпуÑтоезначение получитьта префикÑавтонумерации пропиÑÑŒ пуÑтоезначение разм разобратьпозициюдокумента раÑÑчитатьрегиÑтрына раÑÑчитатьрегиÑтрыпо Ñимв Ñоздатьобъект ÑтатуÑвозврата ÑтрколичеÑтвоÑтрок Ñформироватьпозициюдокумента Ñчетпокоду Ñ‚ÐµÐºÑƒÑ‰ÐµÐµÐ²Ñ€ÐµÐ¼Ñ Ñ‚Ð¸Ð¿Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ñ‚Ð¸Ð¿Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸ÑÑÑ‚Ñ€ уÑтановитьтана уÑтановитьтапо фикÑшаблон шаблон ",d="acos asin atan base64значение base64Ñтрока cos exp log log10 pow sin sqrt tan xmlзначение xmlÑтрока xmlтип xmlтипзнч активноеокно безопаÑныйрежим безопаÑныйрежимразделениÑданных булево ввеÑтидату ввеÑтизначение ввеÑтиÑтроку ввеÑтичиÑло возможноÑтьчтениÑxml Ð²Ð¾Ð¿Ñ€Ð¾Ñ Ð²Ð¾ÑÑтановитьзначение врег выгрузитьжурналрегиÑтрации Ð²Ñ‹Ð¿Ð¾Ð»Ð½Ð¸Ñ‚ÑŒÐ¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚ÐºÑƒÐ¾Ð¿Ð¾Ð²ÐµÑ‰ÐµÐ½Ð¸Ñ Ð²Ñ‹Ð¿Ð¾Ð»Ð½Ð¸Ñ‚ÑŒÐ¿Ñ€Ð¾Ð²ÐµÑ€ÐºÑƒÐ¿Ñ€Ð°Ð²Ð´Ð¾Ñтупа вычиÑлить год данныеформывзначение дата день деньгода деньнедели добавитьмеÑÑц заблокироватьданныедлÑÑ€ÐµÐ´Ð°ÐºÑ‚Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð·Ð°Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²Ð°Ñ‚ÑŒÑ€Ð°Ð±Ð¾Ñ‚ÑƒÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð·Ð°Ð²ÐµÑ€ÑˆÐ¸Ñ‚ÑŒÑ€Ð°Ð±Ð¾Ñ‚ÑƒÑиÑтемы загрузитьвнешнююкомпоненту закрытьÑправку запиÑатьjson запиÑатьxml запиÑатьдатуjson запиÑьжурналарегиÑтрации заполнитьзначениÑÑвойÑтв запроÑÐ¸Ñ‚ÑŒÑ€Ð°Ð·Ñ€ÐµÑˆÐµÐ½Ð¸ÐµÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð·Ð°Ð¿ÑƒÑтитьприложение запуÑтитьÑиÑтему зафикÑироватьтранзакцию значениевданныеформы значениевÑтрокувнутр значениевфайл значениезаполнено значениеизÑтрокивнутр значениеизфайла изxmlтипа импортмоделиxdto имÑкомпьютера имÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð¸Ð½Ð¸Ñ†Ð¸Ð°Ð»Ð¸Ð·Ð¸Ñ€Ð¾Ð²Ð°Ñ‚ÑŒÐ¿Ñ€ÐµÐ´Ð¾Ð¿Ñ€ÐµÐ´ÐµÐ»ÐµÐ½Ð½Ñ‹ÐµÐ´Ð°Ð½Ð½Ñ‹Ðµ информациÑобошибке каталогбиблиотекимобильногоуÑтройÑтва каталогвременныхфайлов каталогдокументов каталогпрограммы кодироватьÑтроку кодлокализацииинформационнойбазы кодÑимвола командаÑиÑтемы конецгода ÐºÐ¾Ð½ÐµÑ†Ð´Ð½Ñ ÐºÐ¾Ð½ÐµÑ†ÐºÐ²Ð°Ñ€Ñ‚Ð°Ð»Ð° конецмеÑÑца конецминуты конецнедели конецчаÑа конфигурациÑбазыданныхизмененадинамичеÑки конфигурациÑизменена копироватьданныеформы копироватьфайл краткоепредÑтавлениеошибки лев Ð¼Ð°ÐºÑ Ð¼ÐµÑÑ‚Ð½Ð¾ÐµÐ²Ñ€ÐµÐ¼Ñ Ð¼ÐµÑÑц мин минута монопольныйрежим найти найтинедопуÑтимыеÑимволыxml найтиокнопонавигационнойÑÑылке найтипомеченныенаудаление найтипоÑÑылкам найтифайлы началогода Ð½Ð°Ñ‡Ð°Ð»Ð¾Ð´Ð½Ñ Ð½Ð°Ñ‡Ð°Ð»Ð¾ÐºÐ²Ð°Ñ€Ñ‚Ð°Ð»Ð° началомеÑÑца началоминуты началонедели началочаÑа начатьзапроÑразрешениÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð½Ð°Ñ‡Ð°Ñ‚ÑŒÐ·Ð°Ð¿ÑƒÑÐºÐ¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð½Ð°Ñ‡Ð°Ñ‚ÑŒÐºÐ¾Ð¿Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸ÐµÑ„Ð°Ð¹Ð»Ð° начатьперемещениефайла начатьподключениевнешнейкомпоненты начатьподключениераÑширениÑработыÑкриптографией начатьподключениераÑширениÑработыÑфайлами начатьпоиÑкфайлов начатьполучениекаталогавременныхфайлов начатьполучениекаталогадокументов Ð½Ð°Ñ‡Ð°Ñ‚ÑŒÐ¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸ÐµÑ€Ð°Ð±Ð¾Ñ‡ÐµÐ³Ð¾ÐºÐ°Ñ‚Ð°Ð»Ð¾Ð³Ð°Ð´Ð°Ð½Ð½Ñ‹Ñ…Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð½Ð°Ñ‡Ð°Ñ‚ÑŒÐ¿Ð¾Ð»ÑƒÑ‡ÐµÐ½Ð¸ÐµÑ„Ð°Ð¹Ð»Ð¾Ð² начатьпомещениефайла начатьпомещениефайлов начатьÑозданиедвоичныхданныхизфайла начатьÑозданиекаталога начатьтранзакцию начатьудалениефайлов начатьуÑтановкувнешнейкомпоненты начатьуÑтановкураÑширениÑработыÑкриптографией начатьуÑтановкураÑширениÑработыÑфайлами неделÑгода необходимоÑтьзавершениÑÑÐ¾ÐµÐ´Ð¸Ð½ÐµÐ½Ð¸Ñ Ð½Ð¾Ð¼ÐµÑ€ÑеанÑаинформационнойбазы номерÑоединениÑинформационнойбазы нрег нÑÑ‚Ñ€ Ð¾Ð±Ð½Ð¾Ð²Ð¸Ñ‚ÑŒÐ¸Ð½Ñ‚ÐµÑ€Ñ„ÐµÐ¹Ñ Ð¾Ð±Ð½Ð¾Ð²Ð¸Ñ‚ÑŒÐ½ÑƒÐ¼ÐµÑ€Ð°Ñ†Ð¸ÑŽÐ¾Ð±ÑŠÐµÐºÑ‚Ð¾Ð² обновитьповторноиÑÐ¿Ð¾Ð»ÑŒÐ·ÑƒÐµÐ¼Ñ‹ÐµÐ·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚ÐºÐ°Ð¿Ñ€ÐµÑ€Ñ‹Ð²Ð°Ð½Ð¸ÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð¾Ð±ÑŠÐµÐ´Ð¸Ð½Ð¸Ñ‚ÑŒÑ„Ð°Ð¹Ð»Ñ‹ окр опиÑаниеошибки оповеÑтить оповеÑтитьобизменении отключитьобработчикзапроÑанаÑÑ‚Ñ€Ð¾ÐµÐºÐºÐ»Ð¸ÐµÐ½Ñ‚Ð°Ð»Ð¸Ñ†ÐµÐ½Ð·Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð¾Ñ‚ÐºÐ»ÑŽÑ‡Ð¸Ñ‚ÑŒÐ¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚Ñ‡Ð¸ÐºÐ¾Ð¶Ð¸Ð´Ð°Ð½Ð¸Ñ Ð¾Ñ‚ÐºÐ»ÑŽÑ‡Ð¸Ñ‚ÑŒÐ¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚Ñ‡Ð¸ÐºÐ¾Ð¿Ð¾Ð²ÐµÑ‰ÐµÐ½Ð¸Ñ Ð¾Ñ‚ÐºÑ€Ñ‹Ñ‚ÑŒÐ·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ðµ открытьиндекÑÑправки открытьÑодержаниеÑправки открытьÑправку открытьформу открытьформумодально отменитьтранзакцию очиÑтитьжурналрегиÑтрации очиÑтитьнаÑÑ‚Ñ€Ð¾Ð¹ÐºÐ¸Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð¾Ñ‡Ð¸ÑтитьÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ñ‹Ð´Ð¾Ñтупа перейтипонавигационнойÑÑылке перемеÑтитьфайл подключитьвнешнююкомпоненту подключитьобработчикзапроÑанаÑÑ‚Ñ€Ð¾ÐµÐºÐºÐ»Ð¸ÐµÐ½Ñ‚Ð°Ð»Ð¸Ñ†ÐµÐ½Ð·Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð¿Ð¾Ð´ÐºÐ»ÑŽÑ‡Ð¸Ñ‚ÑŒÐ¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚Ñ‡Ð¸ÐºÐ¾Ð¶Ð¸Ð´Ð°Ð½Ð¸Ñ Ð¿Ð¾Ð´ÐºÐ»ÑŽÑ‡Ð¸Ñ‚ÑŒÐ¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚Ñ‡Ð¸ÐºÐ¾Ð¿Ð¾Ð²ÐµÑ‰ÐµÐ½Ð¸Ñ Ð¿Ð¾Ð´ÐºÐ»ÑŽÑ‡Ð¸Ñ‚ÑŒÑ€Ð°ÑширениеработыÑкриптографией подключитьраÑширениеработыÑфайлами подробноепредÑтавлениеошибки показатьвводдаты Ð¿Ð¾ÐºÐ°Ð·Ð°Ñ‚ÑŒÐ²Ð²Ð¾Ð´Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð¿Ð¾ÐºÐ°Ð·Ð°Ñ‚ÑŒÐ²Ð²Ð¾Ð´Ñтроки показатьвводчиÑла Ð¿Ð¾ÐºÐ°Ð·Ð°Ñ‚ÑŒÐ²Ð¾Ð¿Ñ€Ð¾Ñ Ð¿Ð¾ÐºÐ°Ð·Ð°Ñ‚ÑŒÐ·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ðµ показатьинформациюобошибке показатьнакарте Ð¿Ð¾ÐºÐ°Ð·Ð°Ñ‚ÑŒÐ¾Ð¿Ð¾Ð²ÐµÑ‰ÐµÐ½Ð¸ÐµÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð¿Ð¾ÐºÐ°Ð·Ð°Ñ‚ÑŒÐ¿Ñ€ÐµÐ´ÑƒÐ¿Ñ€ÐµÐ¶Ð´ÐµÐ½Ð¸Ðµ полноеимÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒcomобъект получитьxmlтип получитьадреÑпомеÑтоположению получитьблокировкуÑеанÑов получитьвремÑзавершениÑÑпÑщегоÑеанÑа получитьвремÑзаÑыпаниÑпаÑÑивногоÑеанÑа получитьвремÑожиданиÑблокировкиданных получитьданныевыбора Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ´Ð¾Ð¿Ð¾Ð»Ð½Ð¸Ñ‚ÐµÐ»ÑŒÐ½Ñ‹Ð¹Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€ÐºÐ»Ð¸ÐµÐ½Ñ‚Ð°Ð»Ð¸Ñ†ÐµÐ½Ð·Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ´Ð¾Ð¿ÑƒÑтимыекодылокализации получитьдопуÑтимыечаÑовыепоÑÑа получитьзаголовокклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ·Ð°Ð³Ð¾Ð»Ð¾Ð²Ð¾ÐºÑиÑтемы получитьзначениÑотборажурналарегиÑтрации получитьидентификаторконфигурации получитьизвременногохранилища получитьимÑвременногофайла получитьимÑÐºÐ»Ð¸ÐµÐ½Ñ‚Ð°Ð»Ð¸Ñ†ÐµÐ½Ð·Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ¸Ð½Ñ„Ð¾Ñ€Ð¼Ð°Ñ†Ð¸ÑŽÑкрановклиента получитьиÑпользованиежурналарегиÑтрации получитьиÑпользованиеÑобытиÑжурналарегиÑтрации Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐºÑ€Ð°Ñ‚ÐºÐ¸Ð¹Ð·Ð°Ð³Ð¾Ð»Ð¾Ð²Ð¾ÐºÐ¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ¼Ð°ÐºÐµÑ‚Ð¾Ñ„Ð¾Ñ€Ð¼Ð»ÐµÐ½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ¼Ð°ÑкувÑефайлы получитьмаÑкувÑефайлыклиента получитьмаÑкувÑефайлыÑервера получитьмеÑтоположениепоадреÑу получитьминимальнуюдлинупаролейпользователей получитьнавигационнуюÑÑылку получитьнавигационнуюÑÑылкуинформационнойбазы получитьобновлениеконфигурациибазыданных получитьобновлениепредопределенныхданныхинформационнойбазы получитьобщиймакет получитьобщуюформу получитьокна получитьоперативнуюотметкувремени получитьотключениебезопаÑногорежима получитьпараметрыфункциональныхопцийинтерфейÑа получитьполноеимÑÐ¿Ñ€ÐµÐ´Ð¾Ð¿Ñ€ÐµÐ´ÐµÐ»ÐµÐ½Ð½Ð¾Ð³Ð¾Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÐ¿Ñ€ÐµÐ´ÑтавлениÑнавигационныхÑÑылок получитьпроверкуÑложноÑтипаролейпользователей получитьразделительпути получитьразделительпутиклиента получитьразделительпутиÑервера получитьÑеанÑыинформационнойбазы получитьÑкороÑтьклиентÑкогоÑÐ¾ÐµÐ´Ð¸Ð½ÐµÐ½Ð¸Ñ Ð¿Ð¾Ð»ÑƒÑ‡Ð¸Ñ‚ÑŒÑоединениÑинформационнойбазы получитьÑообщениÑпользователю получитьÑоответÑтвиеобъектаиформы получитьÑоÑтавÑтандартногоинтерфейÑаodata получитьÑтруктурухранениÑбазыданных получитьтекущийÑеанÑинформационнойбазы получитьфайл получитьфайлы получитьформу получитьфункциональнуюопцию получитьфункциональнуюопциюинтерфейÑа получитьчаÑовойпоÑÑинформационнойбазы Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ð¸Ð¾Ñ Ð¿Ð¾Ð¼ÐµÑтитьвовременноехранилище помеÑтитьфайл помеÑтитьфайлы прав праводоÑтупа предопределенноезначение предÑтавлениекодалокализации предÑтавлениепериода предÑтавлениеправа предÑÑ‚Ð°Ð²Ð»ÐµÐ½Ð¸ÐµÐ¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð¿Ñ€ÐµÐ´ÑтавлениеÑобытиÑжурналарегиÑтрации предÑтавлениечаÑовогопоÑÑа предупреждение прекратитьработуÑиÑтемы привилегированныйрежим продолжитьвызов прочитатьjson прочитатьxml прочитатьдатуjson пуÑтаÑÑтрока Ñ€Ð°Ð±Ð¾Ñ‡Ð¸Ð¹ÐºÐ°Ñ‚Ð°Ð»Ð¾Ð³Ð´Ð°Ð½Ð½Ñ‹Ñ…Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ñ€Ð°Ð·Ð±Ð»Ð¾ÐºÐ¸Ñ€Ð¾Ð²Ð°Ñ‚ÑŒÐ´Ð°Ð½Ð½Ñ‹ÐµÐ´Ð»ÑÑ€ÐµÐ´Ð°ÐºÑ‚Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ñ€Ð°Ð·Ð´ÐµÐ»Ð¸Ñ‚ÑŒÑ„Ð°Ð¹Ð» разорватьÑоединениеÑвнешнимиÑточникомданных раÑкодироватьÑтроку рольдоÑтупна Ñекунда Ñигнал Ñимвол ÑкопироватьжурналрегиÑтрации Ñмещениелетнеговремени ÑмещениеÑтандартноговремени Ñоединитьбуферыдвоичныхданных Ñоздатькаталог Ñоздатьфабрикуxdto Ñокрл Ñокрлп Ñокрп Ñообщить ÑоÑтоÑние Ñохранитьзначение ÑохранитьнаÑÑ‚Ñ€Ð¾Ð¹ÐºÐ¸Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ Ñред Ñтрдлина ÑтрзаканчиваетÑÑна Ñтрзаменить Ñтрнайти ÑтрначинаетÑÑÑ Ñтрока ÑтрокаÑоединениÑинформационнойбазы ÑтрполучитьÑтроку Ñтрразделить ÑÑ‚Ñ€Ñоединить ÑÑ‚Ñ€Ñравнить ÑтрчиÑловхождений ÑтрчиÑлоÑтрок Ñтршаблон текущаÑдата текущаÑдатаÑеанÑа текущаÑуниверÑальнаÑдата текущаÑуниверÑальнаÑдатавмиллиÑекундах текущийвариантинтерфейÑаклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ñ‚ÐµÐºÑƒÑ‰Ð¸Ð¹Ð²Ð°Ñ€Ð¸Ð°Ð½Ñ‚Ð¾ÑновногошрифтаклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ñ‚ÐµÐºÑƒÑ‰Ð¸Ð¹ÐºÐ¾Ð´Ð»Ð¾ÐºÐ°Ð»Ð¸Ð·Ð°Ñ†Ð¸Ð¸ текущийрежимзапуÑка текущийÑзык текущийÑзыкÑиÑтемы тип типзнч транзакциÑактивна трег удалитьданныеинформационнойбазы удалитьизвременногохранилища удалитьобъекты удалитьфайлы универÑÐ°Ð»ÑŒÐ½Ð¾ÐµÐ²Ñ€ÐµÐ¼Ñ ÑƒÑтановитьбезопаÑныйрежим уÑтановитьбезопаÑныйрежимразделениÑданных уÑтановитьблокировкуÑеанÑов уÑтановитьвнешнююкомпоненту уÑтановитьвремÑзавершениÑÑпÑщегоÑеанÑа уÑтановитьвремÑзаÑыпаниÑпаÑÑивногоÑеанÑа уÑтановитьвремÑожиданиÑблокировкиданных уÑтановитьзаголовокклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ ÑƒÑтановитьзаголовокÑиÑтемы уÑтановитьиÑпользованиежурналарегиÑтрации уÑтановитьиÑпользованиеÑобытиÑжурналарегиÑтрации уÑÑ‚Ð°Ð½Ð¾Ð²Ð¸Ñ‚ÑŒÐºÑ€Ð°Ñ‚ÐºÐ¸Ð¹Ð·Ð°Ð³Ð¾Ð»Ð¾Ð²Ð¾ÐºÐ¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ ÑƒÑтановитьминимальнуюдлинупаролейпользователей уÑтановитьмонопольныйрежим уÑтановитьнаÑÑ‚Ñ€Ð¾Ð¹ÐºÐ¸ÐºÐ»Ð¸ÐµÐ½Ñ‚Ð°Ð»Ð¸Ñ†ÐµÐ½Ð·Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ ÑƒÑтановитьобновлениепредопределенныхданныхинформационнойбазы уÑтановитьотключениебезопаÑногорежима уÑтановитьпараметрыфункциональныхопцийинтерфейÑа уÑтановитьпривилегированныйрежим уÑтановитьпроверкуÑложноÑтипаролейпользователей уÑтановитьраÑширениеработыÑкриптографией уÑтановитьраÑширениеработыÑфайлами уÑтановитьÑоединениеÑвнешнимиÑточникомданных уÑтановитьÑоответÑтвиеобъектаиформы уÑтановитьÑоÑтавÑтандартногоинтерфейÑаodata уÑтановитьчаÑовойпоÑÑинформационнойбазы уÑтановитьчаÑовойпоÑÑÑеанÑа формат цел Ñ‡Ð°Ñ Ñ‡Ð°ÑовойпоÑÑ Ñ‡Ð°ÑовойпоÑÑÑеанÑа чиÑло чиÑлопропиÑью ÑтоадреÑвременногохранилища ",p="wsÑÑылки библиотекакартинок библиотекамакетовоформлениÑкомпоновкиданных библиотекаÑтилей бизнеÑпроцеÑÑÑ‹ внешниеиÑточникиданных внешниеобработки внешниеотчеты вÑтроенныепокупки Ð³Ð»Ð°Ð²Ð½Ñ‹Ð¹Ð¸Ð½Ñ‚ÐµÑ€Ñ„ÐµÐ¹Ñ Ð³Ð»Ð°Ð²Ð½Ñ‹Ð¹Ñтиль документы доÑтавлÑÐµÐ¼Ñ‹ÐµÑƒÐ²ÐµÐ´Ð¾Ð¼Ð»ÐµÐ½Ð¸Ñ Ð¶ÑƒÑ€Ð½Ð°Ð»Ñ‹Ð´Ð¾ÐºÑƒÐ¼ÐµÐ½Ñ‚Ð¾Ð² задачи информациÑобинтернетÑоединении иÑпользованиерабочейдаты иÑториÑÑ€Ð°Ð±Ð¾Ñ‚Ñ‹Ð¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ ÐºÐ¾Ð½Ñтанты критерииотбора метаданные обработки отображениерекламы отправкадоÑтавлÑемыхуведомлений отчеты Ð¿Ð°Ð½ÐµÐ»ÑŒÐ·Ð°Ð´Ð°Ñ‡Ð¾Ñ Ð¿Ð°Ñ€Ð°Ð¼ÐµÑ‚Ñ€Ð·Ð°Ð¿ÑƒÑка параметрыÑеанÑа перечиÑÐ»ÐµÐ½Ð¸Ñ Ð¿Ð»Ð°Ð½Ñ‹Ð²Ð¸Ð´Ð¾Ð²Ñ€Ð°Ñчета планывидовхарактериÑтик планыобмена планыÑчетов полнотекÑтовыйпоиÑк пользователиинформационнойбазы поÑледовательноÑти проверкавÑтроенныхпокупок рабочаÑдата раÑширениÑконфигурации региÑтрыбухгалтерии региÑÑ‚Ñ€Ñ‹Ð½Ð°ÐºÐ¾Ð¿Ð»ÐµÐ½Ð¸Ñ Ñ€ÐµÐ³Ð¸ÑтрыраÑчета региÑтрыÑведений Ñ€ÐµÐ³Ð»Ð°Ð¼ÐµÐ½Ñ‚Ð½Ñ‹ÐµÐ·Ð°Ð´Ð°Ð½Ð¸Ñ Ñериализаторxdto Ñправочники ÑредÑÑ‚Ð²Ð°Ð³ÐµÐ¾Ð¿Ð¾Ð·Ð¸Ñ†Ð¸Ð¾Ð½Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ ÑредÑтвакриптографии ÑредÑтвамультимедиа ÑредÑтваотображениÑрекламы ÑредÑтвапочты ÑредÑтвателефонии фабрикаxdto файловыепотоки Ñ„Ð¾Ð½Ð¾Ð²Ñ‹ÐµÐ·Ð°Ð´Ð°Ð½Ð¸Ñ Ñ…Ñ€Ð°Ð½Ð¸Ð»Ð¸Ñ‰Ð°Ð½Ð°Ñтроек хранилищевариантовотчетов хранилищенаÑтроекданныхформ хранилищеобщихнаÑтроек хранилищепользовательÑкихнаÑтроекдинамичеÑкихÑпиÑков хранилищепользовательÑкихнаÑтроекотчетов хранилищеÑиÑтемныхнаÑтроек ",m=l+c+d+p,u="webцвета windowsцвета windowsшрифты библиотекакартинок рамкиÑÑ‚Ð¸Ð»Ñ Ñимволы цветаÑÑ‚Ð¸Ð»Ñ ÑˆÑ€Ð¸Ñ„Ñ‚Ñ‹ÑÑ‚Ð¸Ð»Ñ ",b="автоматичеÑкоеÑохранениеданныхформывнаÑтройках автонумерациÑвформе автораздвижениеÑерий анимациÑдиаграммы вариантвыравниваниÑÑлементовизаголовков вариантуправлениÑвыÑотойтаблицы вертикальнаÑпрокруткаформы вертикальноеположение вертикальноеположениеÑлемента видгруппыформы виддекорацииформы виддополнениÑÑлементаформы видизменениÑданных видкнопкиформы Ð²Ð¸Ð´Ð¿ÐµÑ€ÐµÐºÐ»ÑŽÑ‡Ð°Ñ‚ÐµÐ»Ñ Ð²Ð¸Ð´Ð¿Ð¾Ð´Ð¿Ð¸Ñейкдиаграмме видполÑформы видфлажка влиÑниеразмеранапузырекдиаграммы горизонтальноеположение горизонтальноеположениеÑлемента группировкаколонок группировкаподчиненныхÑлементовформы группыиÑлементы дейÑтвиеперетаÑÐºÐ¸Ð²Ð°Ð½Ð¸Ñ Ð´Ð¾Ð¿Ð¾Ð»Ð½Ð¸Ñ‚ÐµÐ»ÑŒÐ½Ñ‹Ð¹Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ñ‚Ð¾Ð±Ñ€Ð°Ð¶ÐµÐ½Ð¸Ñ Ð´Ð¾Ð¿ÑƒÑтимыедейÑтвиÑперетаÑÐºÐ¸Ð²Ð°Ð½Ð¸Ñ Ð¸Ð½Ñ‚ÐµÑ€Ð²Ð°Ð»Ð¼ÐµÐ¶Ð´ÑƒÑлементамиформы иÑпользованиевывода иÑпользованиеполоÑыпрокрутки иÑпользуемоезначениеточкибиржевойдиаграммы иÑториÑвыборапривводе иÑточникзначенийоÑиточекдиаграммы иÑточникзначениÑразмерапузырькадиаграммы категориÑгруппыкоманд макÑимумÑерий начальноеотображениедерева начальноеотображениеÑпиÑка обновлениетекÑÑ‚Ð°Ñ€ÐµÐ´Ð°ÐºÑ‚Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð¾Ñ€Ð¸ÐµÐ½Ñ‚Ð°Ñ†Ð¸Ñдендрограммы ориентациÑдиаграммы ориентациÑметокдиаграммы ориентациÑметокÑводнойдиаграммы ориентациÑÑлементаформы отображениевдиаграмме отображениевлегендедиаграммы отображениегруппыкнопок отображениезаголовкашкалыдиаграммы отображениезначенийÑводнойдиаграммы отображениезначениÑизмерительнойдиаграммы отображениеинтерваладиаграммыганта отображениекнопки отображениекнопкивыбора отображениеобÑужденийформы отображениеобычнойгруппы отображениеотрицательныхзначенийпузырьковойдиаграммы отображениепанелипоиÑка отображениеподÑказки отображениепредупреждениÑприредактировании отображениеразметкиполоÑÑ‹Ñ€ÐµÐ³ÑƒÐ»Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ð¾Ñ‚Ð¾Ð±Ñ€Ð°Ð¶ÐµÐ½Ð¸ÐµÑтраницформы отображениетаблицы отображениетекÑтазначениÑдиаграммыганта отображениеуправлениÑобычнойгруппы отображениефигурыкнопки палитрацветовдиаграммы поведениеобычнойгруппы поддержкамаÑштабадендрограммы поддержкамаÑштабадиаграммыганта поддержкамаÑштабаÑводнойдиаграммы поиÑквтаблицепривводе положениезаголовкаÑлементаформы положениекартинкикнопкиформы положениекартинкиÑлементаграфичеÑкойÑхемы положениекоманднойпанелиформы положениекоманднойпанелиÑлементаформы положениеопорнойточкиотриÑовки положениеподпиÑейкдиаграмме положениеподпиÑейшкалызначенийизмерительнойдиаграммы положениеÑоÑтоÑниÑпроÑмотра положениеÑтрокипоиÑка положениетекÑтаÑоединительнойлинии положениеуправлениÑпоиÑком положениешкалывремени порÑдокотображениÑточекгоризонтальнойгиÑтограммы порÑдокÑерийвлегендедиаграммы размеркартинки раÑположениезаголовкашкалыдиаграммы раÑÑ‚Ñгиваниеповертикалидиаграммыганта режимавтоотображениÑÑоÑтоÑÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð²Ð²Ð¾Ð´Ð°Ñтроктаблицы режимвыборанезаполненного режимвыделениÑдаты режимвыделениÑÑтрокитаблицы режимвыделениÑтаблицы режимизменениÑразмера режимизменениÑÑвÑÐ·Ð°Ð½Ð½Ð¾Ð³Ð¾Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¸ÑпользованиÑдиалогапечати режимиÑпользованиÑпараметракоманды режиммаÑштабированиÑпроÑмотра режимоÑновногоокнаклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ñ‚ÐºÑ€Ñ‹Ñ‚Ð¸Ñокнаформы режимотображениÑÐ²Ñ‹Ð´ÐµÐ»ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ñ‚Ð¾Ð±Ñ€Ð°Ð¶ÐµÐ½Ð¸ÑгеографичеÑкойÑхемы режимотображениÑзначенийÑерии режимотриÑовкиÑеткиграфичеÑкойÑхемы режимполупрозрачноÑтидиаграммы режимпробеловдиаграммы режимразмещениÑнаÑтранице режимредактированиÑколонки режимÑглаживаниÑдиаграммы режимÑглаживаниÑиндикатора режимÑпиÑказадач Ñквозноевыравнивание ÑохранениеданныхформывнаÑтройках ÑпоÑобзаполнениÑтекÑтазаголовкашкалыдиаграммы ÑпоÑобопределениÑограничивающегозначениÑдиаграммы ÑтандартнаÑгруппакоманд Ñтандартноеоформление ÑтатуÑоповещениÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ñ‚ÐµÐ»Ñ ÑтильÑтрелки типаппрокÑимациилиниитрендадиаграммы типдиаграммы типединицышкалывремени типимпортаÑерийÑлоÑгеографичеÑкойÑхемы типлиниигеографичеÑкойÑхемы типлиниидиаграммы типмаркерагеографичеÑкойÑхемы типмаркерадиаграммы типоблаÑÑ‚Ð¸Ð¾Ñ„Ð¾Ñ€Ð¼Ð»ÐµÐ½Ð¸Ñ Ñ‚Ð¸Ð¿Ð¾Ñ€Ð³Ð°Ð½Ð¸Ð·Ð°Ñ†Ð¸Ð¸Ð¸ÑточникаданныхгеографичеÑкойÑхемы типотображениÑÑерииÑлоÑгеографичеÑкойÑхемы типотображениÑточечногообъектагеографичеÑкойÑхемы типотображениÑшкалыÑлементалегендыгеографичеÑкойÑхемы типпоиÑкаобъектовгеографичеÑкойÑхемы типпроекциигеографичеÑкойÑхемы типразмещениÑизмерений типразмещениÑреквизитовизмерений типрамкиÑÐ»ÐµÐ¼ÐµÐ½Ñ‚Ð°ÑƒÐ¿Ñ€Ð°Ð²Ð»ÐµÐ½Ð¸Ñ Ñ‚Ð¸Ð¿Ñводнойдиаграммы типÑвÑзидиаграммыганта типÑоединениÑзначенийпоÑериÑмдиаграммы типÑоединениÑточекдиаграммы типÑоединительнойлинии типÑтороныÑлементаграфичеÑкойÑхемы типформыотчета типшкалырадарнойдиаграммы факторлиниитрендадиаграммы фигуракнопки фигурыграфичеÑкойÑхемы фикÑациÑвтаблице форматднÑшкалывремени форматкартинки ширинаподчиненныхÑлементовформы ",g="виддвижениÑбухгалтерии виддвижениÑÐ½Ð°ÐºÐ¾Ð¿Ð»ÐµÐ½Ð¸Ñ Ð²Ð¸Ð´Ð¿ÐµÑ€Ð¸Ð¾Ð´Ð°Ñ€ÐµÐ³Ð¸ÑтрараÑчета видÑчета видточкимаршрутабизнеÑпроцеÑÑа иÑпользованиеагрегатарегиÑÑ‚Ñ€Ð°Ð½Ð°ÐºÐ¾Ð¿Ð»ÐµÐ½Ð¸Ñ Ð¸ÑпользованиегруппиÑлементов иÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ð½Ð¸ÐµÑ€ÐµÐ¶Ð¸Ð¼Ð°Ð¿Ñ€Ð¾Ð²ÐµÐ´ÐµÐ½Ð¸Ñ Ð¸ÑпользованиеÑреза периодичноÑтьагрегатарегиÑÑ‚Ñ€Ð°Ð½Ð°ÐºÐ¾Ð¿Ð»ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð°Ð²Ñ‚Ð¾Ð²Ñ€ÐµÐ¼Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð·Ð°Ð¿Ð¸Ñидокумента режимпроведениÑдокумента ",f="авторегиÑтрациÑизменений допуÑтимыйномерÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ Ð¾Ñ‚Ð¿Ñ€Ð°Ð²ÐºÐ°Ñлементаданных получениеÑлементаданных ",_="иÑпользованиераÑшифровкитабличногодокумента ориентациÑÑтраницы положениеитоговколонокÑводнойтаблицы положениеитоговÑтрокÑводнойтаблицы положениетекÑтаотноÑительнокартинки раÑположениезаголовкагруппировкитабличногодокумента ÑпоÑобчтениÑзначенийтабличногодокумента типдвуÑтороннейпечати типзаполнениÑоблаÑтитабличногодокумента типкурÑоровтабличногодокумента типлиниириÑункатабличногодокумента типлинииÑчейкитабличногодокумента типнаправлениÑпереходатабличногодокумента типотображениÑвыделениÑтабличногодокумента типотображениÑлинийÑводнойтаблицы типразмещениÑтекÑтатабличногодокумента типриÑункатабличногодокумента типÑмещениÑтабличногодокумента типузоратабличногодокумента типфайлатабличногодокумента точноÑтьпечати чередованиераÑположениÑÑтраниц ",h="отображениевремениÑлементовпланировщика ",v="типфайлаформатированногодокумента ",y="обходрезультатазапроÑа типзапиÑизапроÑа ",S="видзаполнениÑраÑшифровкипоÑтроителÑотчета типдобавлениÑпредÑтавлений типизмерениÑпоÑтроителÑотчета типразмещениÑитогов ",C="доÑтупкфайлу режимдиалогавыборафайла режимоткрытиÑфайла ",x="типизмерениÑпоÑтроителÑзапроÑа ",E="видданныханализа методклаÑтеризации типединицыинтервалавременианализаданных типзаполнениÑтаблицырезультатаанализаданных типиÑпользованиÑчиÑловыхзначенийанализаданных типиÑточникаданныхпоиÑкааÑÑоциаций типколонкианализаданныхдереворешений типколонкианализаданныхклаÑÑ‚ÐµÑ€Ð¸Ð·Ð°Ñ†Ð¸Ñ Ñ‚Ð¸Ð¿ÐºÐ¾Ð»Ð¾Ð½ÐºÐ¸Ð°Ð½Ð°Ð»Ð¸Ð·Ð°Ð´Ð°Ð½Ð½Ñ‹Ñ…Ð¾Ð±Ñ‰Ð°ÑÑтатиÑтика типколонкианализаданныхпоиÑкаÑÑоциаций типколонкианализаданныхпоиÑкпоÑледовательноÑтей типколонкимоделипрогноза типмерыраÑÑтоÑниÑанализаданных типотÑечениÑправилаÑÑоциации типполÑанализаданных типÑтандартизациианализаданных типупорÑдочиваниÑправилаÑÑоциациианализаданных типупорÑдочиваниÑшаблоновпоÑледовательноÑтейанализаданных типупрощениÑдереварешений ",N="wsнаправлениепараметра вариантxpathxs вариантзапиÑидатыjson вариантпроÑтоготипаxs видгруппымоделиxs видфаÑетаxdto дейÑтвиепоÑтроителÑdom завершенноÑтьпроÑтоготипаxs завершенноÑÑ‚ÑŒÑоÑтавноготипаxs завершенноÑÑ‚ÑŒÑхемыxs запрещенныеподÑтановкиxs иÑключениÑгруппподÑтановкиxs категориÑиÑпользованиÑатрибутаxs категориÑограничениÑидентичноÑтиxs категориÑограничениÑпроÑтранÑтвименxs методнаÑледованиÑxs модельÑодержимогоxs назначениетипаxml недопуÑтимыеподÑтановкиxs обработкапробельныхÑимволовxs обработкаÑодержимогоxs ограничениезначениÑxs параметрыотбораузловdom переноÑÑтрокjson позициÑвдокументеdom пробельныеÑимволыxml типатрибутаxml типзначениÑjson типканоничеÑкогоxml типкомпонентыxs типпроверкиxml типрезультатаdomxpath типузлаdom типузлаxml формаxml формапредÑтавлениÑxs форматдатыjson ÑкранированиеÑимволовjson ",T="видÑравнениÑкомпоновкиданных дейÑтвиеобработкираÑшифровкикомпоновкиданных направлениеÑортировкикомпоновкиданных раÑположениевложенныхÑлементоврезультатакомпоновкиданных раÑположениеитоговкомпоновкиданных раÑположениегруппировкикомпоновкиданных раÑположениеполейгруппировкикомпоновкиданных раÑположениеполÑкомпоновкиданных раÑположениереквизитовкомпоновкиданных раÑположениереÑурÑовкомпоновкиданных типбухгалтерÑкогооÑтаткакомпоновкиданных типвыводатекÑтакомпоновкиданных типгруппировкикомпоновкиданных типгруппыÑлементовотборакомпоновкиданных типдополнениÑпериодакомпоновкиданных типзаголовкаполейкомпоновкиданных типмакетагруппировкикомпоновкиданных типмакетаоблаÑтикомпоновкиданных типоÑтаткакомпоновкиданных типпериодакомпоновкиданных типразмещениÑтекÑтакомпоновкиданных типÑвÑзинаборовданныхкомпоновкиданных типÑлементарезультатакомпоновкиданных раÑположениелегендыдиаграммыкомпоновкиданных типприменениÑотборакомпоновкиданных режимотображениÑÑлементанаÑтройкикомпоновкиданных режимотображениÑнаÑтроеккомпоновкиданных ÑоÑтоÑниеÑлементанаÑтройкикомпоновкиданных ÑпоÑобвоÑÑтановлениÑнаÑтроеккомпоновкиданных режимкомпоновкирезультата иÑпользованиепараметракомпоновкиданных автопозициÑреÑурÑовкомпоновкиданных вариантиÑпользованиÑгруппировкикомпоновкиданных раÑположениереÑурÑоввдиаграммекомпоновкиданных фикÑациÑкомпоновкиданных иÑпользованиеуÑловногооформлениÑкомпоновкиданных ",w="важноÑтьинтернетпочтовогоÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ Ð¾Ð±Ñ€Ð°Ð±Ð¾Ñ‚ÐºÐ°Ñ‚ÐµÐºÑтаинтернетпочтовогоÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ ÑпоÑобкодированиÑÐ¸Ð½Ñ‚ÐµÑ€Ð½ÐµÑ‚Ð¿Ð¾Ñ‡Ñ‚Ð¾Ð²Ð¾Ð³Ð¾Ð²Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ ÑпоÑобкодированиÑнеasciiÑимволовинтернетпочтовогоÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ Ñ‚Ð¸Ð¿Ñ‚ÐµÐºÑтапочтовогоÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ Ð¿Ñ€Ð¾Ñ‚Ð¾ÐºÐ¾Ð»Ð¸Ð½Ñ‚ÐµÑ€Ð½ÐµÑ‚Ð¿Ð¾Ñ‡Ñ‚Ñ‹ ÑтатуÑразборапочтовогоÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ ",A="режимтранзакциизапиÑижурналарегиÑтрации ÑтатуÑтранзакциизапиÑижурналарегиÑтрации уровеньжурналарегиÑтрации ",M="раÑположениехранилищаÑертификатовкриптографии режимвключениÑÑертификатовкриптографии режимпроверкиÑертификатакриптографии типхранилищаÑертификатовкриптографии ",D="кодировкаименфайловвzipфайле методÑжатиÑzip методшифрованиÑzip режимвоÑÑтановлениÑпутейфайловzip режимобработкиподкаталоговzip режимÑохранениÑпутейzip уровеньÑжатиÑzip ",I="звуковоеоповещение направлениепереходакÑтроке позициÑвпотоке порÑдокбайтов режимблокировкиданных режимуправлениÑблокировкойданных ÑервиÑвÑтроенныхпокупок ÑоÑтоÑÐ½Ð¸ÐµÑ„Ð¾Ð½Ð¾Ð²Ð¾Ð³Ð¾Ð·Ð°Ð´Ð°Ð½Ð¸Ñ Ñ‚Ð¸Ð¿Ð¿Ð¾Ð´Ð¿Ð¸ÑчикадоÑтавлÑемыхуведомлений уровеньиÑпользованиÑзащищенногоÑоединениÑftp ",k="направлениепорÑдкаÑхемызапроÑа типдополнениÑпериодамиÑхемызапроÑа типконтрольнойточкиÑхемызапроÑа типобъединениÑÑхемызапроÑа типпараметрадоÑтупнойтаблицыÑхемызапроÑа типÑоединениÑÑхемызапроÑа ",R="httpметод автоиÑпользованиеобщегореквизита автопрефикÑномеразадачи вариантвÑтроенногоÑзыка видиерархии видрегиÑÑ‚Ñ€Ð°Ð½Ð°ÐºÐ¾Ð¿Ð»ÐµÐ½Ð¸Ñ Ð²Ð¸Ð´Ñ‚Ð°Ð±Ð»Ð¸Ñ†Ñ‹Ð²Ð½ÐµÑˆÐ½ÐµÐ³Ð¾Ð¸Ñточникаданных запиÑьдвиженийприпроведении заполнениепоÑледовательноÑтей индекÑирование иÑпользованиебазыпланавидовраÑчета иÑпользованиебыÑтроговыбора иÑпользованиеобщегореквизита иÑÐ¿Ð¾Ð»ÑŒÐ·Ð¾Ð²Ð°Ð½Ð¸ÐµÐ¿Ð¾Ð´Ñ‡Ð¸Ð½ÐµÐ½Ð¸Ñ Ð¸ÑпользованиеполнотекÑтовогопоиÑка иÑпользованиеразделÑемыхданныхобщегореквизита иÑпользованиереквизита назначениеиÑпользованиÑÐ¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð½Ð°Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸ÐµÑ€Ð°ÑширениÑконфигурации направлениепередачи обновлениепредопределенныхданных оперативноепроведение оÑновноепредÑтавлениевидараÑчета оÑновноепредÑтавлениевидахарактериÑтики оÑновноепредÑтавлениезадачи оÑновноепредÑтавлениепланаобмена оÑновноепредÑтавлениеÑправочника оÑновноепредÑтавлениеÑчета перемещениеграницыприпроведении периодичноÑтьномерабизнеÑпроцеÑÑа периодичноÑтьномерадокумента периодичноÑтьрегиÑтрараÑчета периодичноÑтьрегиÑтраÑведений повторноеиÑпользованиевозвращаемыхзначений полнотекÑтовыйпоиÑкпривводепоÑтроке принадлежноÑтьобъекта проведение разделениеаутентификацииобщегореквизита разделениеданныхобщегореквизита разделениераÑширенийконфигурацииобщегореквизита режимавтонумерацииобъектов режимзапиÑирегиÑтра режимиÑпользованиÑмодальноÑти режимиÑпользованиÑÑинхронныхвызововраÑширенийплатформыивнешнихкомпонент режимповторногоиÑпользованиÑÑеанÑов режимполучениÑданныхвыборапривводепоÑтроке режимÑовмеÑтимоÑти режимÑовмеÑтимоÑтиинтерфейÑа режимуправлениÑблокировкойданныхпоумолчанию ÑериикодовпланавидовхарактериÑтик ÑериикодовпланаÑчетов ÑериикодовÑправочника Ñозданиепривводе ÑпоÑобвыбора ÑпоÑобпоиÑкаÑтрокипривводепоÑтроке ÑпоÑÐ¾Ð±Ñ€ÐµÐ´Ð°ÐºÑ‚Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸Ñ Ñ‚Ð¸Ð¿Ð´Ð°Ð½Ð½Ñ‹Ñ…Ñ‚Ð°Ð±Ð»Ð¸Ñ†Ñ‹Ð²Ð½ÐµÑˆÐ½ÐµÐ³Ð¾Ð¸Ñточникаданных типкодапланавидовраÑчета типкодаÑправочника типмакета типномерабизнеÑпроцеÑÑа типномерадокумента типномеразадачи типформы удалениедвижений ",L="важноÑтьпроблемыприменениÑраÑширениÑконфигурации вариантинтерфейÑаклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð²Ð°Ñ€Ð¸Ð°Ð½Ñ‚Ð¼Ð°ÑштабаформклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð²Ð°Ñ€Ð¸Ð°Ð½Ñ‚Ð¾ÑновногошрифтаклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ð²Ð°Ñ€Ð¸Ð°Ð½Ñ‚Ñтандартногопериода вариантÑтандартнойдатыначала видграницы видкартинки видотображениÑполнотекÑтовогопоиÑка видрамки видÑÑ€Ð°Ð²Ð½ÐµÐ½Ð¸Ñ Ð²Ð¸Ð´Ñ†Ð²ÐµÑ‚Ð° видчиÑÐ»Ð¾Ð²Ð¾Ð³Ð¾Ð·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ð²Ð¸Ð´ÑˆÑ€Ð¸Ñ„Ñ‚Ð° допуÑтимаÑдлина допуÑтимыйзнак иÑпользованиеbyteordermark иÑпользованиеметаданныхполнотекÑтовогопоиÑка иÑточникраÑширенийконфигурации клавиша кодвозвратадиалога кодировкаxbase кодировкатекÑта направлениепоиÑка направлениеÑортировки обновлениепредопределенныхданных обновлениеприизмененииданных отображениепанелиразделов Ð¿Ñ€Ð¾Ð²ÐµÑ€ÐºÐ°Ð·Ð°Ð¿Ð¾Ð»Ð½ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð´Ð¸Ð°Ð»Ð¾Ð³Ð°Ð²Ð¾Ð¿Ñ€Ð¾Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð·Ð°Ð¿ÑƒÑкаклиентÑÐºÐ¾Ð³Ð¾Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾ÐºÑ€ÑƒÐ³Ð»ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¾Ñ‚ÐºÑ€Ñ‹Ñ‚Ð¸ÑÑ„Ð¾Ñ€Ð¼Ð¿Ñ€Ð¸Ð»Ð¾Ð¶ÐµÐ½Ð¸Ñ Ñ€ÐµÐ¶Ð¸Ð¼Ð¿Ð¾Ð»Ð½Ð¾Ñ‚ÐµÐºÑтовогопоиÑка ÑкороÑтьклиентÑкогоÑÐ¾ÐµÐ´Ð¸Ð½ÐµÐ½Ð¸Ñ ÑоÑтоÑниевнешнегоиÑточникаданных ÑоÑтоÑниеобновлениÑконфигурациибазыданных ÑпоÑобвыбораÑертификатаwindows ÑпоÑобкодированиÑÑтроки ÑтатуÑÑÐ¾Ð¾Ð±Ñ‰ÐµÐ½Ð¸Ñ Ñ‚Ð¸Ð¿Ð²Ð½ÐµÑˆÐ½ÐµÐ¹ÐºÐ¾Ð¼Ð¿Ð¾Ð½ÐµÐ½Ñ‚Ñ‹ типплатформы типповедениÑклавишиenter типÑлементаинформацииовыполненииобновлениÑконфигурациибазыданных уровеньизолÑциитранзакций Ñ…ÐµÑˆÑ„ÑƒÐ½ÐºÑ†Ð¸Ñ Ñ‡Ð°Ñтидаты",P=u+b+g+f+_+h+v+y+S+C+x+E+N+T+w+A+M+D+I+k+R+L,O="comобъект ftpÑоединение httpÐ·Ð°Ð¿Ñ€Ð¾Ñ httpÑервиÑответ httpÑоединение wsÐ¾Ð¿Ñ€ÐµÐ´ÐµÐ»ÐµÐ½Ð¸Ñ wsпрокÑи xbase анализданных аннотациÑxs блокировкаданных буфердвоичныхданных включениеxs выражениекомпоновкиданных генераторÑлучайныхчиÑел географичеÑкаÑÑхема географичеÑкиекоординаты графичеÑкаÑÑхема группамоделиxs данныераÑшифровкикомпоновкиданных двоичныеданные дендрограмма диаграмма диаграммаганта диалогвыборафайла диалогвыборацвета диалогвыборашрифта диалограÑпиÑаниÑÑ€ÐµÐ³Ð»Ð°Ð¼ÐµÐ½Ñ‚Ð½Ð¾Ð³Ð¾Ð·Ð°Ð´Ð°Ð½Ð¸Ñ Ð´Ð¸Ð°Ð»Ð¾Ð³Ñ€ÐµÐ´Ð°ÐºÑ‚Ð¸Ñ€Ð¾Ð²Ð°Ð½Ð¸ÑÑтандартногопериода диапазон документdom документhtml документациÑxs доÑтавлÑемоеуведомление запиÑÑŒdom запиÑÑŒfastinfoset запиÑÑŒhtml запиÑÑŒjson запиÑÑŒxml запиÑÑŒzipфайла запиÑьданных запиÑьтекÑта запиÑьузловdom Ð·Ð°Ð¿Ñ€Ð¾Ñ Ð·Ð°Ñ‰Ð¸Ñ‰ÐµÐ½Ð½Ð¾ÐµÑоединениеopenssl значениÑполейраÑшифровкикомпоновкиданных извлечениетекÑта импортxs интернетпочта интернетпочтовоеÑообщение интернетпочтовыйпрофиль интернетпрокÑи интернетÑоединение информациÑдлÑприложениÑxs иÑпользованиеатрибутаxs иÑпользованиеÑобытиÑжурналарегиÑтрации иÑточникдоÑтупныхнаÑтроеккомпоновкиданных итераторузловdom картинка квалификаторыдаты квалификаторыдвоичныхданных квалификаторыÑтроки квалификаторычиÑла компоновщикмакетакомпоновкиданных компоновщикнаÑтроеккомпоновкиданных конÑтруктормакетаоформлениÑкомпоновкиданных конÑтрукторнаÑтроеккомпоновкиданных конÑтрукторформатнойÑтроки Ð»Ð¸Ð½Ð¸Ñ Ð¼Ð°ÐºÐµÑ‚ÐºÐ¾Ð¼Ð¿Ð¾Ð½Ð¾Ð²ÐºÐ¸Ð´Ð°Ð½Ð½Ñ‹Ñ… макетоблаÑтикомпоновкиданных макетоформлениÑкомпоновкиданных маÑкаxs менеджеркриптографии наборÑхемxml наÑтройкикомпоновкиданных наÑтройкиÑериализацииjson обработкакартинок обработкараÑшифровкикомпоновкиданных обходдереваdom объÑвлениеатрибутаxs объÑвлениенотацииxs объÑвлениеÑлементаxs опиÑаниеиÑпользованиÑÑобытиÑдоÑтупжурналарегиÑтрации опиÑаниеиÑпользованиÑÑобытиÑотказвдоÑтупежурналарегиÑтрации опиÑаниеобработкираÑшифровкикомпоновкиданных опиÑаниепередаваемогофайла опиÑаниетипов определениегруппыатрибутовxs определениегруппымоделиxs определениеограничениÑидентичноÑтиxs определениепроÑтоготипаxs определениеÑоÑтавноготипаxs определениетипадокументаdom определениÑxpathxs отборкомпоновкиданных пакетотображаемыхдокументов параметрвыбора параметркомпоновкиданных параметрызапиÑиjson параметрызапиÑиxml параметрычтениÑxml переопределениеxs планировщик полеанализаданных полекомпоновкиданных поÑтроительdom поÑтроительзапроÑа поÑтроительотчета поÑтроительотчетаанализаданных поÑтроительÑхемxml поток потоквпамÑти почта почтовоеÑообщение преобразованиеxsl преобразованиекканоничеÑкомуxml процеÑÑорвыводарезультатакомпоновкиданныхвколлекциюзначений процеÑÑорвыводарезультатакомпоновкиданныхвтабличныйдокумент процеÑÑоркомпоновкиданных разыменовательпроÑтранÑтвименdom рамка раÑпиÑÐ°Ð½Ð¸ÐµÑ€ÐµÐ³Ð»Ð°Ð¼ÐµÐ½Ñ‚Ð½Ð¾Ð³Ð¾Ð·Ð°Ð´Ð°Ð½Ð¸Ñ Ñ€Ð°ÑширенноеимÑxml результатчтениÑданных ÑводнаÑдиаграмма ÑвÑзьпараметравыбора ÑвÑзьпотипу ÑвÑзьпотипукомпоновкиданных Ñериализаторxdto Ñертификатклиентаwindows Ñертификатклиентафайл Ñертификаткриптографии ÑертификатыудоÑтоверÑющихцентровwindows ÑертификатыудоÑтоверÑющихцентровфайл Ñжатиеданных ÑиÑтемнаÑÐ¸Ð½Ñ„Ð¾Ñ€Ð¼Ð°Ñ†Ð¸Ñ Ñообщениепользователю Ñочетаниеклавиш Ñравнениезначений ÑтандартнаÑдатаначала Ñтандартныйпериод Ñхемаxml Ñхемакомпоновкиданных табличныйдокумент текÑтовыйдокумент теÑтируемоеприложение типданныхxml уникальныйидентификатор фабрикаxdto файл файловыйпоток фаÑетдлиныxs фаÑетколичеÑтваразрÑдовдробнойчаÑтиxs фаÑетмакÑимальноговключающегозначениÑxs фаÑетмакÑимальногоиÑключающегозначениÑxs фаÑетмакÑимальнойдлиныxs фаÑетминимальноговключающегозначениÑxs фаÑетминимальногоиÑключающегозначениÑxs фаÑетминимальнойдлиныxs фаÑетобразцаxs фаÑетобщегоколичеÑтваразрÑдовxs фаÑетперечиÑлениÑxs фаÑетпробельныхÑимволовxs фильтрузловdom форматированнаÑÑтрока форматированныйдокумент фрагментxs хешированиеданных Ñ…Ñ€Ð°Ð½Ð¸Ð»Ð¸Ñ‰ÐµÐ·Ð½Ð°Ñ‡ÐµÐ½Ð¸Ñ Ñ†Ð²ÐµÑ‚ чтениеfastinfoset чтениеhtml чтениеjson чтениеxml чтениеzipфайла чтениеданных чтениетекÑта чтениеузловdom шрифт Ñлементрезультатакомпоновкиданных ",F="comsafearray деревозначений маÑÑив ÑоответÑтвие ÑпиÑокзначений Ñтруктура таблицазначений фикÑированнаÑÑтруктура фикÑированноеÑоответÑтвие фикÑированныймаÑÑив ",B=O+F,G="null иÑтина ложь неопределено",q=e.inherit(e.NM),U={cN:"string",b:'"|\\|',e:'"|$',c:[{b:'""'}]},z={b:"'",e:"'",eB:!0,eE:!0,c:[{cN:"number",b:"\\d{4}([\\.\\\\/:-]?\\d{2}){0,5}"}]},$=e.inherit(e.CLCM),V={cN:"meta",l:t,b:"#|&",e:"$",k:{"meta-keyword":i+s},c:[$]},W={cN:"symbol",b:"~",e:";|:",eE:!0},H={cN:"function",l:t,v:[{b:"процедура|функциÑ",e:"\\)",k:"процедура функциÑ"},{b:"конецпроцедуры|конецфункции",k:"конецпроцедуры конецфункции"}],c:[{b:"\\(",e:"\\)",endsParent:!0,c:[{cN:"params",l:t,b:t,e:",",eE:!0,eW:!0,k:{keyword:"знач",literal:G},c:[q,U,z]},$]},e.inherit(e.TM,{b:t})]};return{cI:!0,l:t,k:{keyword:i,built_in:m,class:P,type:B,literal:G},c:[V,H,$,W,q,U,z]}})),e.registerLanguage("abnf",(function(e){var t={ruleDeclaration:"^[a-zA-Z][a-zA-Z0-9-]*",unexpectedChars:"[!@#$^&',?+~`|:]"},r=["ALPHA","BIT","CHAR","CR","CRLF","CTL","DIGIT","DQUOTE","HEXDIG","HTAB","LF","LWSP","OCTET","SP","VCHAR","WSP"],a=e.C(";","$"),i={cN:"symbol",b:/%b[0-1]+(-[0-1]+|(\.[0-1]+)+){0,1}/},n={cN:"symbol",b:/%d[0-9]+(-[0-9]+|(\.[0-9]+)+){0,1}/},o={cN:"symbol",b:/%x[0-9A-F]+(-[0-9A-F]+|(\.[0-9A-F]+)+){0,1}/},s={cN:"symbol",b:/%[si]/},l={b:t.ruleDeclaration+"\\s*=",rB:!0,e:/=/,r:0,c:[{cN:"attribute",b:t.ruleDeclaration}]};return{i:t.unexpectedChars,k:r.join(" "),c:[l,a,i,n,o,s,e.QSM,e.NM]}})),e.registerLanguage("accesslog",(function(e){return{c:[{cN:"number",b:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{cN:"number",b:"\\b\\d+\\b",r:0},{cN:"string",b:'"(GET|POST|HEAD|PUT|DELETE|CONNECT|OPTIONS|PATCH|TRACE)',e:'"',k:"GET POST HEAD PUT DELETE CONNECT OPTIONS PATCH TRACE",i:"\\n",r:10},{cN:"string",b:/\[/,e:/\]/,i:"\\n"},{cN:"string",b:'"',e:'"',i:"\\n"}]}})),e.registerLanguage("actionscript",(function(e){var t="[a-zA-Z_$][a-zA-Z0-9_$]*",r="([*]|[a-zA-Z_$][a-zA-Z0-9_$]*)",a={cN:"rest_arg",b:"[.]{3}",e:t,r:10};return{aliases:["as"],k:{keyword:"as break case catch class const continue default delete do dynamic each else extends final finally for function get if implements import in include instanceof interface internal is namespace native new override package private protected public return set static super switch this throw try typeof use var void while with",literal:"true false null undefined"},c:[e.ASM,e.QSM,e.CLCM,e.CBCM,e.CNM,{cN:"class",bK:"package",e:"{",c:[e.TM]},{cN:"class",bK:"class interface",e:"{",eE:!0,c:[{bK:"extends implements"},e.TM]},{cN:"meta",bK:"import include",e:";",k:{"meta-keyword":"import include"}},{cN:"function",bK:"function",e:"[{;]",eE:!0,i:"\\S",c:[e.TM,{cN:"params",b:"\\(",e:"\\)",c:[e.ASM,e.QSM,e.CLCM,e.CBCM,a]},{b:":\\s*"+r}]},e.METHOD_GUARD],i:/#/}})),e.registerLanguage("ada",(function(e){var t="\\d(_|\\d)*",r="[eE][-+]?"+t,a=t+"(\\."+t+")?("+r+")?",i="\\w+",n=t+"#"+i+"(\\."+i+")?#("+r+")?",o="\\b("+n+"|"+a+")",s="[A-Za-z](_?[A-Za-z0-9.])*",l="[]{}%#'\"",c=e.C("--","$"),d={b:"\\s+:\\s+",e:"\\s*(:=|;|\\)|=>|$)",i:l,c:[{bK:"loop for declare others",endsParent:!0},{cN:"keyword",bK:"not null constant access function procedure in out aliased exception"},{cN:"type",b:s,endsParent:!0,r:0}]};return{cI:!0,k:{keyword:"abort else new return abs elsif not reverse abstract end accept entry select access exception of separate aliased exit or some all others subtype and for out synchronized array function overriding at tagged generic package task begin goto pragma terminate body private then if procedure type case in protected constant interface is raise use declare range delay limited record when delta loop rem while digits renames with do mod requeue xor",literal:"True False"},c:[c,{cN:"string",b:/"/,e:/"/,c:[{b:/""/,r:0}]},{cN:"string",b:/'.'/},{cN:"number",b:o,r:0},{cN:"symbol",b:"'"+s},{cN:"title",b:"(\\bwith\\s+)?(\\bprivate\\s+)?\\bpackage\\s+(\\bbody\\s+)?",e:"(is|$)",k:"package body",eB:!0,eE:!0,i:l},{b:"(\\b(with|overriding)\\s+)?\\b(function|procedure)\\s+",e:"(\\bis|\\bwith|\\brenames|\\)\\s*;)",k:"overriding function procedure with is renames return",rB:!0,c:[c,{cN:"title",b:"(\\bwith\\s+)?\\b(function|procedure)\\s+",e:"(\\(|\\s+|$)",eB:!0,eE:!0,i:l},d,{cN:"type",b:"\\breturn\\s+",e:"(\\s+|;|$)",k:"return",eB:!0,eE:!0,endsParent:!0,i:l}]},{cN:"type",b:"\\b(sub)?type\\s+",e:"\\s+",k:"type",eB:!0,i:l},d]}})),e.registerLanguage("apache",(function(e){var t={cN:"number",b:"[\\$%]\\d+"};return{aliases:["apacheconf"],cI:!0,c:[e.HCM,{cN:"section",b:""},{cN:"attribute",b:/\w+/,r:0,k:{nomarkup:"order deny allow setenv rewriterule rewriteengine rewritecond documentroot sethandler errordocument loadmodule options header listen serverroot servername"},starts:{e:/$/,r:0,k:{literal:"on off all"},c:[{cN:"meta",b:"\\s\\[",e:"\\]$"},{cN:"variable",b:"[\\$%]\\{",e:"\\}",c:["self",t]},t,e.QSM]}}],i:/\S/}})),e.registerLanguage("applescript",(function(e){var t=e.inherit(e.QSM,{i:""}),r={cN:"params",b:"\\(",e:"\\)",c:["self",e.CNM,t]},a=e.C("--","$"),i=e.C("\\(\\*","\\*\\)",{c:["self",a]}),n=[a,i,e.HCM];return{aliases:["osascript"],k:{keyword:"about above after against and around as at back before beginning behind below beneath beside between but by considering contain contains continue copy div does eighth else end equal equals error every exit fifth first for fourth from front get given global if ignoring in into is it its last local me middle mod my ninth not of on onto or over prop property put ref reference repeat returning script second set seventh since sixth some tell tenth that the|0 then third through thru timeout times to transaction try until where while whose with without",literal:"AppleScript false linefeed return pi quote result space tab true",built_in:"alias application boolean class constant date file integer list number real record string text activate beep count delay launch log offset read round run say summarize write character characters contents day frontmost id item length month name paragraph paragraphs rest reverse running time version weekday word words year"},c:[t,e.CNM,{cN:"built_in",b:"\\b(clipboard info|the clipboard|info for|list (disks|folder)|mount volume|path to|(close|open for) access|(get|set) eof|current date|do shell script|get volume settings|random number|set volume|system attribute|system info|time to GMT|(load|run|store) script|scripting components|ASCII (character|number)|localized string|choose (application|color|file|file name|folder|from list|remote application|URL)|display (alert|dialog))\\b|^\\s*return\\b"},{cN:"literal",b:"\\b(text item delimiters|current application|missing value)\\b"},{cN:"keyword",b:"\\b(apart from|aside from|instead of|out of|greater than|isn't|(doesn't|does not) (equal|come before|come after|contain)|(greater|less) than( or equal)?|(starts?|ends|begins?) with|contained by|comes (before|after)|a (ref|reference)|POSIX file|POSIX path|(date|time) string|quoted form)\\b"},{bK:"on",i:"[${=;\\n]",c:[e.UTM,r]}].concat(n),i:"//|->|=>|\\[\\["}})),e.registerLanguage("cpp",(function(e){var t={cN:"keyword",b:"\\b[a-z\\d_]*_t\\b"},r={cN:"string",v:[{b:'(u8?|U)?L?"',e:'"',i:"\\n",c:[e.BE]},{b:'(u8?|U)?R"',e:'"',c:[e.BE]},{b:"'\\\\?.",e:"'",i:"."}]},a={cN:"number",v:[{b:"\\b(0b[01']+)"},{b:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{b:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],r:0},i={cN:"meta",b:/#\s*[a-z]+\b/,e:/$/,k:{"meta-keyword":"if else elif endif define undef warning error line pragma ifdef ifndef include"},c:[{b:/\\\n/,r:0},e.inherit(r,{cN:"meta-string"}),{cN:"meta-string",b:/<[^\n>]*>/,e:/$/,i:"\\n"},e.CLCM,e.CBCM]},n=e.IR+"\\s*\\(",o={keyword:"int float while private char catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignof constexpr decltype noexcept static_assert thread_local restrict _Bool complex _Complex _Imaginary atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and or not",built_in:"std string cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap array shared_ptr abort abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr",literal:"true false nullptr NULL"},s=[t,e.CLCM,e.CBCM,a,r];return{aliases:["c","cc","h","c++","h++","hpp"],k:o,i:"",k:o,c:["self",t]},{b:e.IR+"::",k:o},{v:[{b:/=/,e:/;/},{b:/\(/,e:/\)/},{bK:"new throw return else",e:/;/}],k:o,c:s.concat([{b:/\(/,e:/\)/,k:o,c:s.concat(["self"]),r:0}]),r:0},{cN:"function",b:"("+e.IR+"[\\*&\\s]+)+"+n,rB:!0,e:/[{;=]/,eE:!0,k:o,i:/[^\w\s\*&]/,c:[{b:n,rB:!0,c:[e.TM],r:0},{cN:"params",b:/\(/,e:/\)/,k:o,r:0,c:[e.CLCM,e.CBCM,r,a,t]},e.CLCM,e.CBCM,i]},{cN:"class",bK:"class struct",e:/[{;:]/,c:[{b://,c:["self"]},e.TM]}]),exports:{preprocessor:i,strings:r,k:o}}})),e.registerLanguage("arduino",(function(e){var t=e.getLanguage("cpp").exports;return{k:{keyword:"boolean byte word string String array "+t.k.keyword,built_in:"setup loop while catch for if do goto try switch case else default break continue return KeyboardController MouseController SoftwareSerial EthernetServer EthernetClient LiquidCrystal RobotControl GSMVoiceCall EthernetUDP EsploraTFT HttpClient RobotMotor WiFiClient GSMScanner FileSystem Scheduler GSMServer YunClient YunServer IPAddress GSMClient GSMModem Keyboard Ethernet Console GSMBand Esplora Stepper Process WiFiUDP GSM_SMS Mailbox USBHost Firmata PImage Client Server GSMPIN FileIO Bridge Serial EEPROM Stream Mouse Audio Servo File Task GPRS WiFi Wire TFT GSM SPI SD runShellCommandAsynchronously analogWriteResolution retrieveCallingNumber printFirmwareVersion analogReadResolution sendDigitalPortPair noListenOnLocalhost readJoystickButton setFirmwareVersion readJoystickSwitch scrollDisplayRight getVoiceCallStatus scrollDisplayLeft writeMicroseconds delayMicroseconds beginTransmission getSignalStrength runAsynchronously getAsynchronously listenOnLocalhost getCurrentCarrier readAccelerometer messageAvailable sendDigitalPorts lineFollowConfig countryNameWrite runShellCommand readStringUntil rewindDirectory readTemperature setClockDivider readLightSensor endTransmission analogReference detachInterrupt countryNameRead attachInterrupt encryptionType readBytesUntil robotNameWrite readMicrophone robotNameRead cityNameWrite userNameWrite readJoystickY readJoystickX mouseReleased openNextFile scanNetworks noInterrupts digitalWrite beginSpeaker mousePressed isActionDone mouseDragged displayLogos noAutoscroll addParameter remoteNumber getModifiers keyboardRead userNameRead waitContinue processInput parseCommand printVersion readNetworks writeMessage blinkVersion cityNameRead readMessage setDataMode parsePacket isListening setBitOrder beginPacket isDirectory motorsWrite drawCompass digitalRead clearScreen serialEvent rightToLeft setTextSize leftToRight requestFrom keyReleased compassRead analogWrite interrupts WiFiServer disconnect playMelody parseFloat autoscroll getPINUsed setPINUsed setTimeout sendAnalog readSlider analogRead beginWrite createChar motorsStop keyPressed tempoWrite readButton subnetMask debugPrint macAddress writeGreen randomSeed attachGPRS readString sendString remotePort releaseAll mouseMoved background getXChange getYChange answerCall getResult voiceCall endPacket constrain getSocket writeJSON getButton available connected findUntil readBytes exitValue readGreen writeBlue startLoop IPAddress isPressed sendSysex pauseMode gatewayIP setCursor getOemKey tuneWrite noDisplay loadImage switchPIN onRequest onReceive changePIN playFile noBuffer parseInt overflow checkPIN knobRead beginTFT bitClear updateIR bitWrite position writeRGB highByte writeRed setSpeed readBlue noStroke remoteIP transfer shutdown hangCall beginSMS endWrite attached maintain noCursor checkReg checkPUK shiftOut isValid shiftIn pulseIn connect println localIP pinMode getIMEI display noBlink process getBand running beginSD drawBMP lowByte setBand release bitRead prepare pointTo readRed setMode noFill remove listen stroke detach attach noTone exists buffer height bitSet circle config cursor random IRread setDNS endSMS getKey micros millis begin print write ready flush width isPIN blink clear press mkdir rmdir close point yield image BSSID click delay read text move peek beep rect line open seek fill size turn stop home find step tone sqrt RSSI SSID end bit tan cos sin pow map abs max min get run put",literal:"DIGITAL_MESSAGE FIRMATA_STRING ANALOG_MESSAGE REPORT_DIGITAL REPORT_ANALOG INPUT_PULLUP SET_PIN_MODE INTERNAL2V56 SYSTEM_RESET LED_BUILTIN INTERNAL1V1 SYSEX_START INTERNAL EXTERNAL DEFAULT OUTPUT INPUT HIGH LOW"},c:[t.preprocessor,e.CLCM,e.CBCM,e.ASM,e.QSM,e.CNM]}})),e.registerLanguage("armasm",(function(e){return{cI:!0,aliases:["arm"],l:"\\.?"+e.IR,k:{meta:".2byte .4byte .align .ascii .asciz .balign .byte .code .data .else .end .endif .endm .endr .equ .err .exitm .extern .global .hword .if .ifdef .ifndef .include .irp .long .macro .rept .req .section .set .skip .space .text .word .arm .thumb .code16 .code32 .force_thumb .thumb_func .ltorg ALIAS ALIGN ARM AREA ASSERT ATTR CN CODE CODE16 CODE32 COMMON CP DATA DCB DCD DCDU DCDO DCFD DCFDU DCI DCQ DCQU DCW DCWU DN ELIF ELSE END ENDFUNC ENDIF ENDP ENTRY EQU EXPORT EXPORTAS EXTERN FIELD FILL FUNCTION GBLA GBLL GBLS GET GLOBAL IF IMPORT INCBIN INCLUDE INFO KEEP LCLA LCLL LCLS LTORG MACRO MAP MEND MEXIT NOFP OPT PRESERVE8 PROC QN READONLY RELOC REQUIRE REQUIRE8 RLIST FN ROUT SETA SETL SETS SN SPACE SUBT THUMB THUMBX TTL WHILE WEND ",built_in:"r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 pc lr sp ip sl sb fp a1 a2 a3 a4 v1 v2 v3 v4 v5 v6 v7 v8 f0 f1 f2 f3 f4 f5 f6 f7 p0 p1 p2 p3 p4 p5 p6 p7 p8 p9 p10 p11 p12 p13 p14 p15 c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 c10 c11 c12 c13 c14 c15 q0 q1 q2 q3 q4 q5 q6 q7 q8 q9 q10 q11 q12 q13 q14 q15 cpsr_c cpsr_x cpsr_s cpsr_f cpsr_cx cpsr_cxs cpsr_xs cpsr_xsf cpsr_sf cpsr_cxsf spsr_c spsr_x spsr_s spsr_f spsr_cx spsr_cxs spsr_xs spsr_xsf spsr_sf spsr_cxsf s0 s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15 s16 s17 s18 s19 s20 s21 s22 s23 s24 s25 s26 s27 s28 s29 s30 s31 d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d11 d12 d13 d14 d15 d16 d17 d18 d19 d20 d21 d22 d23 d24 d25 d26 d27 d28 d29 d30 d31 {PC} {VAR} {TRUE} {FALSE} {OPT} {CONFIG} {ENDIAN} {CODESIZE} {CPU} {FPU} {ARCHITECTURE} {PCSTOREOFFSET} {ARMASM_VERSION} {INTER} {ROPI} {RWPI} {SWST} {NOSWST} . @"},c:[{cN:"keyword",b:"\\b(adc|(qd?|sh?|u[qh]?)?add(8|16)?|usada?8|(q|sh?|u[qh]?)?(as|sa)x|and|adrl?|sbc|rs[bc]|asr|b[lx]?|blx|bxj|cbn?z|tb[bh]|bic|bfc|bfi|[su]bfx|bkpt|cdp2?|clz|clrex|cmp|cmn|cpsi[ed]|cps|setend|dbg|dmb|dsb|eor|isb|it[te]{0,3}|lsl|lsr|ror|rrx|ldm(([id][ab])|f[ds])?|ldr((s|ex)?[bhd])?|movt?|mvn|mra|mar|mul|[us]mull|smul[bwt][bt]|smu[as]d|smmul|smmla|mla|umlaal|smlal?([wbt][bt]|d)|mls|smlsl?[ds]|smc|svc|sev|mia([bt]{2}|ph)?|mrr?c2?|mcrr2?|mrs|msr|orr|orn|pkh(tb|bt)|rbit|rev(16|sh)?|sel|[su]sat(16)?|nop|pop|push|rfe([id][ab])?|stm([id][ab])?|str(ex)?[bhd]?|(qd?)?sub|(sh?|q|u[qh]?)?sub(8|16)|[su]xt(a?h|a?b(16)?)|srs([id][ab])?|swpb?|swi|smi|tst|teq|wfe|wfi|yield)(eq|ne|cs|cc|mi|pl|vs|vc|hi|ls|ge|lt|gt|le|al|hs|lo)?[sptrx]?",e:"\\s"},e.C("[;@]","$",{r:0}),e.CBCM,e.QSM,{cN:"string",b:"'",e:"[^\\\\]'",r:0},{cN:"title",b:"\\|",e:"\\|",i:"\\n",r:0},{cN:"number",v:[{b:"[#$=]?0x[0-9a-f]+"},{b:"[#$=]?0b[01]+"},{b:"[#$=]\\d+"},{b:"\\b\\d+"}],r:0},{cN:"symbol",v:[{b:"^[a-z_\\.\\$][a-z0-9_\\.\\$]+"},{b:"^\\s*[a-z_\\.\\$][a-z0-9_\\.\\$]+:"},{b:"[=#]\\w+"}],r:0}]}})),e.registerLanguage("xml",(function(e){var t="[A-Za-z0-9\\._:-]+",r={eW:!0,i:/`]+/}]}]}]};return{aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist"],cI:!0,c:[{cN:"meta",b:"",r:10,c:[{b:"\\[",e:"\\]"}]},e.C("\x3c!--","--\x3e",{r:10}),{b:"<\\!\\[CDATA\\[",e:"\\]\\]>",r:10},{b:/<\?(php)?/,e:/\?>/,sL:"php",c:[{b:"/\\*",e:"\\*/",skip:!0}]},{cN:"tag",b:"|$)",e:">",k:{name:"style"},c:[r],starts:{e:"",rE:!0,sL:["css","xml"]}},{cN:"tag",b:"|$)",e:">",k:{name:"script"},c:[r],starts:{e:"<\/script>",rE:!0,sL:["actionscript","javascript","handlebars","xml"]}},{cN:"meta",v:[{b:/<\?xml/,e:/\?>/,r:10},{b:/<\?\w+/,e:/\?>/}]},{cN:"tag",b:"",c:[{cN:"name",b:/[^\/><\s]+/,r:0},r]}]}})),e.registerLanguage("asciidoc",(function(e){return{aliases:["adoc"],c:[e.C("^/{4,}\\n","\\n/{4,}$",{r:10}),e.C("^//","$",{r:0}),{cN:"title",b:"^\\.\\w.*$"},{b:"^[=\\*]{4,}\\n",e:"\\n^[=\\*]{4,}$",r:10},{cN:"section",r:10,v:[{b:"^(={1,5}) .+?( \\1)?$"},{b:"^[^\\[\\]\\n]+?\\n[=\\-~\\^\\+]{2,}$"}]},{cN:"meta",b:"^:.+?:",e:"\\s",eE:!0,r:10},{cN:"meta",b:"^\\[.+?\\]$",r:0},{cN:"quote",b:"^_{4,}\\n",e:"\\n_{4,}$",r:10},{cN:"code",b:"^[\\-\\.]{4,}\\n",e:"\\n[\\-\\.]{4,}$",r:10},{b:"^\\+{4,}\\n",e:"\\n\\+{4,}$",c:[{b:"<",e:">",sL:"xml",r:0}],r:10},{cN:"bullet",b:"^(\\*+|\\-+|\\.+|[^\\n]+?::)\\s+"},{cN:"symbol",b:"^(NOTE|TIP|IMPORTANT|WARNING|CAUTION):\\s+",r:10},{cN:"strong",b:"\\B\\*(?![\\*\\s])",e:"(\\n{2}|\\*)",c:[{b:"\\\\*\\w",r:0}]},{cN:"emphasis",b:"\\B'(?!['\\s])",e:"(\\n{2}|')",c:[{b:"\\\\'\\w",r:0}],r:0},{cN:"emphasis",b:"_(?![_\\s])",e:"(\\n{2}|_)",r:0},{cN:"string",v:[{b:"``.+?''"},{b:"`.+?'"}]},{cN:"code",b:"(`.+?`|\\+.+?\\+)",r:0},{cN:"code",b:"^[ \\t]",e:"$",r:0},{b:"^'{3,}[ \\t]*$",r:10},{b:"(link:)?(http|https|ftp|file|irc|image:?):\\S+\\[.*?\\]",rB:!0,c:[{b:"(link|image:?):",r:0},{cN:"link",b:"\\w",e:"[^\\[]+",r:0},{cN:"string",b:"\\[",e:"\\]",eB:!0,eE:!0,r:0}],r:10}]}})),e.registerLanguage("aspectj",(function(e){var t="false synchronized int abstract float private char boolean static null if const for true while long throw strictfp finally protected import native final return void enum else extends implements break transient new catch instanceof byte super volatile case assert short package default double public try this switch continue throws privileged aspectOf adviceexecution proceed cflowbelow cflow initialization preinitialization staticinitialization withincode target within execution getWithinTypeName handler thisJoinPoint thisJoinPointStaticPart thisEnclosingJoinPointStaticPart declare parents warning error soft precedence thisAspectInstance",r="get set args call";return{k:t,i:/<\/|#/,c:[e.C("/\\*\\*","\\*/",{r:0,c:[{b:/\w+@/,r:0},{cN:"doctag",b:"@[A-Za-z]+"}]}),e.CLCM,e.CBCM,e.ASM,e.QSM,{cN:"class",bK:"aspect",e:/[{;=]/,eE:!0,i:/[:;"\[\]]/,c:[{bK:"extends implements pertypewithin perthis pertarget percflowbelow percflow issingleton"},e.UTM,{b:/\([^\)]*/,e:/[)]+/,k:t+" "+r,eE:!1}]},{cN:"class",bK:"class interface",e:/[{;=]/,eE:!0,r:0,k:"class interface",i:/[:"\[\]]/,c:[{bK:"extends implements"},e.UTM]},{bK:"pointcut after before around throwing returning",e:/[)]/,eE:!1,i:/["\[\]]/,c:[{b:e.UIR+"\\s*\\(",rB:!0,c:[e.UTM]}]},{b:/[:]/,rB:!0,e:/[{;]/,r:0,eE:!1,k:t,i:/["\[\]]/,c:[{b:e.UIR+"\\s*\\(",k:t+" "+r,r:0},e.QSM]},{bK:"new throw",r:0},{cN:"function",b:/\w+ +\w+(\.)?\w+\s*\([^\)]*\)\s*((throws)[\w\s,]+)?[\{;]/,rB:!0,e:/[{;=]/,k:t,eE:!0,c:[{b:e.UIR+"\\s*\\(",rB:!0,r:0,c:[e.UTM]},{cN:"params",b:/\(/,e:/\)/,r:0,k:t,c:[e.ASM,e.QSM,e.CNM,e.CBCM]},e.CLCM,e.CBCM]},e.CNM,{cN:"meta",b:"@[A-Za-z]+"}]}})),e.registerLanguage("autohotkey",(function(e){var t={b:"`[\\s\\S]"};return{cI:!0,aliases:["ahk"],k:{keyword:"Break Continue Critical Exit ExitApp Gosub Goto New OnExit Pause return SetBatchLines SetTimer Suspend Thread Throw Until ahk_id ahk_class ahk_pid ahk_exe ahk_group",literal:"A|0 true false NOT AND OR",built_in:"ComSpec Clipboard ClipboardAll ErrorLevel"},c:[{cN:"built_in",b:"A_[a-zA-Z0-9]+"},t,e.inherit(e.QSM,{c:[t]}),e.C(";","$",{r:0}),e.CBCM,{cN:"number",b:e.NR,r:0},{cN:"subst",b:"%(?=[a-zA-Z0-9#_$@])",e:"%",i:"[^a-zA-Z0-9#_$@]"},{cN:"built_in",b:"^\\s*\\w+\\s*,"},{cN:"meta",b:"^\\s*#w+",e:"$",r:0},{cN:"symbol",c:[t],v:[{b:'^[^\\n";]+::(?!=)'},{b:'^[^\\n";]+:(?!=)',r:0}]},{b:",\\s*,"}]}})),e.registerLanguage("autoit",(function(e){var t="ByRef Case Const ContinueCase ContinueLoop Default Dim Do Else ElseIf EndFunc EndIf EndSelect EndSwitch EndWith Enum Exit ExitLoop For Func Global If In Local Next ReDim Return Select Static Step Switch Then To Until Volatile WEnd While With",r="True False And Null Not Or",a="Abs ACos AdlibRegister AdlibUnRegister Asc AscW ASin Assign ATan AutoItSetOption AutoItWinGetTitle AutoItWinSetTitle Beep Binary BinaryLen BinaryMid BinaryToString BitAND BitNOT BitOR BitRotate BitShift BitXOR BlockInput Break Call CDTray Ceiling Chr ChrW ClipGet ClipPut ConsoleRead ConsoleWrite ConsoleWriteError ControlClick ControlCommand ControlDisable ControlEnable ControlFocus ControlGetFocus ControlGetHandle ControlGetPos ControlGetText ControlHide ControlListView ControlMove ControlSend ControlSetText ControlShow ControlTreeView Cos Dec DirCopy DirCreate DirGetSize DirMove DirRemove DllCall DllCallAddress DllCallbackFree DllCallbackGetPtr DllCallbackRegister DllClose DllOpen DllStructCreate DllStructGetData DllStructGetPtr DllStructGetSize DllStructSetData DriveGetDrive DriveGetFileSystem DriveGetLabel DriveGetSerial DriveGetType DriveMapAdd DriveMapDel DriveMapGet DriveSetLabel DriveSpaceFree DriveSpaceTotal DriveStatus EnvGet EnvSet EnvUpdate Eval Execute Exp FileChangeDir FileClose FileCopy FileCreateNTFSLink FileCreateShortcut FileDelete FileExists FileFindFirstFile FileFindNextFile FileFlush FileGetAttrib FileGetEncoding FileGetLongName FileGetPos FileGetShortcut FileGetShortName FileGetSize FileGetTime FileGetVersion FileInstall FileMove FileOpen FileOpenDialog FileRead FileReadLine FileReadToArray FileRecycle FileRecycleEmpty FileSaveDialog FileSelectFolder FileSetAttrib FileSetEnd FileSetPos FileSetTime FileWrite FileWriteLine Floor FtpSetProxy FuncName GUICreate GUICtrlCreateAvi GUICtrlCreateButton GUICtrlCreateCheckbox GUICtrlCreateCombo GUICtrlCreateContextMenu GUICtrlCreateDate GUICtrlCreateDummy GUICtrlCreateEdit GUICtrlCreateGraphic GUICtrlCreateGroup GUICtrlCreateIcon GUICtrlCreateInput GUICtrlCreateLabel GUICtrlCreateList GUICtrlCreateListView GUICtrlCreateListViewItem GUICtrlCreateMenu GUICtrlCreateMenuItem GUICtrlCreateMonthCal GUICtrlCreateObj GUICtrlCreatePic GUICtrlCreateProgress GUICtrlCreateRadio GUICtrlCreateSlider GUICtrlCreateTab GUICtrlCreateTabItem GUICtrlCreateTreeView GUICtrlCreateTreeViewItem GUICtrlCreateUpdown GUICtrlDelete GUICtrlGetHandle GUICtrlGetState GUICtrlRead GUICtrlRecvMsg GUICtrlRegisterListViewSort GUICtrlSendMsg GUICtrlSendToDummy GUICtrlSetBkColor GUICtrlSetColor GUICtrlSetCursor GUICtrlSetData GUICtrlSetDefBkColor GUICtrlSetDefColor GUICtrlSetFont GUICtrlSetGraphic GUICtrlSetImage GUICtrlSetLimit GUICtrlSetOnEvent GUICtrlSetPos GUICtrlSetResizing GUICtrlSetState GUICtrlSetStyle GUICtrlSetTip GUIDelete GUIGetCursorInfo GUIGetMsg GUIGetStyle GUIRegisterMsg GUISetAccelerators GUISetBkColor GUISetCoord GUISetCursor GUISetFont GUISetHelp GUISetIcon GUISetOnEvent GUISetState GUISetStyle GUIStartGroup GUISwitch Hex HotKeySet HttpSetProxy HttpSetUserAgent HWnd InetClose InetGet InetGetInfo InetGetSize InetRead IniDelete IniRead IniReadSection IniReadSectionNames IniRenameSection IniWrite IniWriteSection InputBox Int IsAdmin IsArray IsBinary IsBool IsDeclared IsDllStruct IsFloat IsFunc IsHWnd IsInt IsKeyword IsNumber IsObj IsPtr IsString Log MemGetStats Mod MouseClick MouseClickDrag MouseDown MouseGetCursor MouseGetPos MouseMove MouseUp MouseWheel MsgBox Number ObjCreate ObjCreateInterface ObjEvent ObjGet ObjName OnAutoItExitRegister OnAutoItExitUnRegister Ping PixelChecksum PixelGetColor PixelSearch ProcessClose ProcessExists ProcessGetStats ProcessList ProcessSetPriority ProcessWait ProcessWaitClose ProgressOff ProgressOn ProgressSet Ptr Random RegDelete RegEnumKey RegEnumVal RegRead RegWrite Round Run RunAs RunAsWait RunWait Send SendKeepActive SetError SetExtended ShellExecute ShellExecuteWait Shutdown Sin Sleep SoundPlay SoundSetWaveVolume SplashImageOn SplashOff SplashTextOn Sqrt SRandom StatusbarGetText StderrRead StdinWrite StdioClose StdoutRead String StringAddCR StringCompare StringFormat StringFromASCIIArray StringInStr StringIsAlNum StringIsAlpha StringIsASCII StringIsDigit StringIsFloat StringIsInt StringIsLower StringIsSpace StringIsUpper StringIsXDigit StringLeft StringLen StringLower StringMid StringRegExp StringRegExpReplace StringReplace StringReverse StringRight StringSplit StringStripCR StringStripWS StringToASCIIArray StringToBinary StringTrimLeft StringTrimRight StringUpper Tan TCPAccept TCPCloseSocket TCPConnect TCPListen TCPNameToIP TCPRecv TCPSend TCPShutdown, UDPShutdown TCPStartup, UDPStartup TimerDiff TimerInit ToolTip TrayCreateItem TrayCreateMenu TrayGetMsg TrayItemDelete TrayItemGetHandle TrayItemGetState TrayItemGetText TrayItemSetOnEvent TrayItemSetState TrayItemSetText TraySetClick TraySetIcon TraySetOnEvent TraySetPauseIcon TraySetState TraySetToolTip TrayTip UBound UDPBind UDPCloseSocket UDPOpen UDPRecv UDPSend VarGetType WinActivate WinActive WinClose WinExists WinFlash WinGetCaretPos WinGetClassList WinGetClientSize WinGetHandle WinGetPos WinGetProcess WinGetState WinGetText WinGetTitle WinKill WinList WinMenuSelectItem WinMinimizeAll WinMinimizeAllUndo WinMove WinSetOnTop WinSetState WinSetTitle WinSetTrans WinWait",i={v:[e.C(";","$",{r:0}),e.C("#cs","#ce"),e.C("#comments-start","#comments-end")]},n={b:"\\$[A-z0-9_]+"},o={cN:"string",v:[{b:/"/,e:/"/,c:[{b:/""/,r:0}]},{b:/'/,e:/'/,c:[{b:/''/,r:0}]}]},s={v:[e.BNM,e.CNM]},l={cN:"meta",b:"#",e:"$",k:{"meta-keyword":"comments include include-once NoTrayIcon OnAutoItStartRegister pragma compile RequireAdmin"},c:[{b:/\\\n/,r:0},{bK:"include",k:{"meta-keyword":"include"},e:"$",c:[o,{cN:"meta-string",v:[{b:"<",e:">"},{b:/"/,e:/"/,c:[{b:/""/,r:0}]},{b:/'/,e:/'/,c:[{b:/''/,r:0}]}]}]},o,i]},c={cN:"symbol",b:"@[A-z0-9_]+"},d={cN:"function",bK:"Func",e:"$",i:"\\$|\\[|%",c:[e.UTM,{cN:"params",b:"\\(",e:"\\)",c:[n,o,s]}]};return{cI:!0,i:/\/\*/,k:{keyword:t,built_in:a,literal:r},c:[i,n,o,s,l,c,d]}})),e.registerLanguage("avrasm",(function(e){return{cI:!0,l:"\\.?"+e.IR,k:{keyword:"adc add adiw and andi asr bclr bld brbc brbs brcc brcs break breq brge brhc brhs brid brie brlo brlt brmi brne brpl brsh brtc brts brvc brvs bset bst call cbi cbr clc clh cli cln clr cls clt clv clz com cp cpc cpi cpse dec eicall eijmp elpm eor fmul fmuls fmulsu icall ijmp in inc jmp ld ldd ldi lds lpm lsl lsr mov movw mul muls mulsu neg nop or ori out pop push rcall ret reti rjmp rol ror sbc sbr sbrc sbrs sec seh sbi sbci sbic sbis sbiw sei sen ser ses set sev sez sleep spm st std sts sub subi swap tst wdr",built_in:"r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r10 r11 r12 r13 r14 r15 r16 r17 r18 r19 r20 r21 r22 r23 r24 r25 r26 r27 r28 r29 r30 r31 x|0 xh xl y|0 yh yl z|0 zh zl ucsr1c udr1 ucsr1a ucsr1b ubrr1l ubrr1h ucsr0c ubrr0h tccr3c tccr3a tccr3b tcnt3h tcnt3l ocr3ah ocr3al ocr3bh ocr3bl ocr3ch ocr3cl icr3h icr3l etimsk etifr tccr1c ocr1ch ocr1cl twcr twdr twar twsr twbr osccal xmcra xmcrb eicra spmcsr spmcr portg ddrg ping portf ddrf sreg sph spl xdiv rampz eicrb eimsk gimsk gicr eifr gifr timsk tifr mcucr mcucsr tccr0 tcnt0 ocr0 assr tccr1a tccr1b tcnt1h tcnt1l ocr1ah ocr1al ocr1bh ocr1bl icr1h icr1l tccr2 tcnt2 ocr2 ocdr wdtcr sfior eearh eearl eedr eecr porta ddra pina portb ddrb pinb portc ddrc pinc portd ddrd pind spdr spsr spcr udr0 ucsr0a ucsr0b ubrr0l acsr admux adcsr adch adcl porte ddre pine pinf",meta:".byte .cseg .db .def .device .dseg .dw .endmacro .equ .eseg .exit .include .list .listmac .macro .nolist .org .set"},c:[e.CBCM,e.C(";","$",{r:0}),e.CNM,e.BNM,{cN:"number",b:"\\b(\\$[a-zA-Z0-9]+|0o[0-7]+)"},e.QSM,{cN:"string",b:"'",e:"[^\\\\]'",i:"[^\\\\][^']"},{cN:"symbol",b:"^[A-Za-z0-9_.$]+:"},{cN:"meta",b:"#",e:"$"},{cN:"subst",b:"@[0-9]+"}]}})),e.registerLanguage("awk",(function(e){var t={cN:"variable",v:[{b:/\$[\w\d#@][\w\d_]*/},{b:/\$\{(.*?)}/}]},r="BEGIN END if else while do for in break continue delete next nextfile function func exit|10",a={cN:"string",c:[e.BE],v:[{b:/(u|b)?r?'''/,e:/'''/,r:10},{b:/(u|b)?r?"""/,e:/"""/,r:10},{b:/(u|r|ur)'/,e:/'/,r:10},{b:/(u|r|ur)"/,e:/"/,r:10},{b:/(b|br)'/,e:/'/},{b:/(b|br)"/,e:/"/},e.ASM,e.QSM]};return{k:{keyword:r},c:[t,a,e.RM,e.HCM,e.NM]}})),e.registerLanguage("axapta",(function(e){return{k:"false int abstract private char boolean static null if for true while long throw finally protected final return void enum else break new catch byte super case short default double public try this switch continue reverse firstfast firstonly forupdate nofetch sum avg minof maxof count order group by asc desc index hint like dispaly edit client server ttsbegin ttscommit str real date container anytype common div mod",c:[e.CLCM,e.CBCM,e.ASM,e.QSM,e.CNM,{cN:"meta",b:"#",e:"$"},{cN:"class",bK:"class interface",e:"{",eE:!0,i:":",c:[{bK:"extends implements"},e.UTM]}]}})),e.registerLanguage("bash",(function(e){var t={cN:"variable",v:[{b:/\$[\w\d#@][\w\d_]*/},{b:/\$\{(.*?)}/}]},r={cN:"string",b:/"/,e:/"/,c:[e.BE,t,{cN:"variable",b:/\$\(/,e:/\)/,c:[e.BE]}]},a={cN:"string",b:/'/,e:/'/};return{aliases:["sh","zsh"],l:/\b-?[a-z\._]+\b/,k:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},c:[{cN:"meta",b:/^#![^\n]+sh\s*$/,r:10},{cN:"function",b:/\w[\w\d_]*\s*\(\s*\)\s*\{/,rB:!0,c:[e.inherit(e.TM,{b:/\w[\w\d_]*/})],r:0},e.HCM,r,a,t]}})),e.registerLanguage("basic",(function(e){return{cI:!0,i:"^.",l:"[a-zA-Z][a-zA-Z0-9_$%!#]*",k:{keyword:"ABS ASC AND ATN AUTO|0 BEEP BLOAD|10 BSAVE|10 CALL CALLS CDBL CHAIN CHDIR CHR$|10 CINT CIRCLE CLEAR CLOSE CLS COLOR COM COMMON CONT COS CSNG CSRLIN CVD CVI CVS DATA DATE$ DEFDBL DEFINT DEFSNG DEFSTR DEF|0 SEG USR DELETE DIM DRAW EDIT END ENVIRON ENVIRON$ EOF EQV ERASE ERDEV ERDEV$ ERL ERR ERROR EXP FIELD FILES FIX FOR|0 FRE GET GOSUB|10 GOTO HEX$ IF|0 THEN ELSE|0 INKEY$ INP INPUT INPUT# INPUT$ INSTR IMP INT IOCTL IOCTL$ KEY ON OFF LIST KILL LEFT$ LEN LET LINE LLIST LOAD LOC LOCATE LOF LOG LPRINT USING LSET MERGE MID$ MKDIR MKD$ MKI$ MKS$ MOD NAME NEW NEXT NOISE NOT OCT$ ON OR PEN PLAY STRIG OPEN OPTION BASE OUT PAINT PALETTE PCOPY PEEK PMAP POINT POKE POS PRINT PRINT] PSET PRESET PUT RANDOMIZE READ REM RENUM RESET|0 RESTORE RESUME RETURN|0 RIGHT$ RMDIR RND RSET RUN SAVE SCREEN SGN SHELL SIN SOUND SPACE$ SPC SQR STEP STICK STOP STR$ STRING$ SWAP SYSTEM TAB TAN TIME$ TIMER TROFF TRON TO USR VAL VARPTR VARPTR$ VIEW WAIT WHILE WEND WIDTH WINDOW WRITE XOR"},c:[e.QSM,e.C("REM","$",{r:10}),e.C("'","$",{r:0}),{cN:"symbol",b:"^[0-9]+ ",r:10},{cN:"number",b:"\\b([0-9]+[0-9edED.]*[#!]?)",r:0},{cN:"number",b:"(&[hH][0-9a-fA-F]{1,4})"},{cN:"number",b:"(&[oO][0-7]{1,6})"}]}})),e.registerLanguage("bnf",(function(e){return{c:[{cN:"attribute",b://},{b:/::=/,starts:{e:/$/,c:[{b://},e.CLCM,e.CBCM,e.ASM,e.QSM]}}]}})),e.registerLanguage("brainfuck",(function(e){var t={cN:"literal",b:"[\\+\\-]",r:0};return{aliases:["bf"],c:[e.C("[^\\[\\]\\.,\\+\\-<> \r\n]","[\\[\\]\\.,\\+\\-<> \r\n]",{rE:!0,r:0}),{cN:"title",b:"[\\[\\]]",r:0},{cN:"string",b:"[\\.,]",r:0},{b:/\+\+|\-\-/,rB:!0,c:[t]},t]}})),e.registerLanguage("cal",(function(e){var t="div mod in and or not xor asserterror begin case do downto else end exit for if of repeat then to until while with var",r="false true",a=[e.CLCM,e.C(/\{/,/\}/,{r:0}),e.C(/\(\*/,/\*\)/,{r:10})],i={cN:"string",b:/'/,e:/'/,c:[{b:/''/}]},n={cN:"string",b:/(#\d+)+/},o={cN:"number",b:"\\b\\d+(\\.\\d+)?(DT|D|T)",r:0},s={cN:"string",b:'"',e:'"'},l={cN:"function",bK:"procedure",e:/[:;]/,k:"procedure|10",c:[e.TM,{cN:"params",b:/\(/,e:/\)/,k:t,c:[i,n]}].concat(a)},c={cN:"class",b:"OBJECT (Table|Form|Report|Dataport|Codeunit|XMLport|MenuSuite|Page|Query) (\\d+) ([^\\r\\n]+)",rB:!0,c:[e.TM,l]};return{cI:!0,k:{keyword:t,literal:r},i:/\/\*/,c:[i,n,o,s,e.NM,c,l]}})),e.registerLanguage("capnproto",(function(e){return{aliases:["capnp"],k:{keyword:"struct enum interface union group import using const annotation extends in of on as with from fixed",built_in:"Void Bool Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64 Float32 Float64 Text Data AnyPointer AnyStruct Capability List",literal:"true false"},c:[e.QSM,e.NM,e.HCM,{cN:"meta",b:/@0x[\w\d]{16};/,i:/\n/},{cN:"symbol",b:/@\d+\b/},{cN:"class",bK:"struct enum",e:/\{/,i:/\n/,c:[e.inherit(e.TM,{starts:{eW:!0,eE:!0}})]},{cN:"class",bK:"interface",e:/\{/,i:/\n/,c:[e.inherit(e.TM,{starts:{eW:!0,eE:!0}})]}]}})),e.registerLanguage("ceylon",(function(e){var t="assembly module package import alias class interface object given value assign void function new of extends satisfies abstracts in out return break continue throw assert dynamic if else switch case for while try catch finally then let this outer super is exists nonempty",r="shared abstract formal default actual variable late native deprecatedfinal sealed annotation suppressWarnings small",a="doc by license see throws tagged",i={cN:"subst",eB:!0,eE:!0,b:/``/,e:/``/,k:t,r:10},n=[{cN:"string",b:'"""',e:'"""',r:10},{cN:"string",b:'"',e:'"',c:[i]},{cN:"string",b:"'",e:"'"},{cN:"number",b:"#[0-9a-fA-F_]+|\\$[01_]+|[0-9_]+(?:\\.[0-9_](?:[eE][+-]?\\d+)?)?[kMGTPmunpf]?",r:0}];return i.c=n,{k:{keyword:t+" "+r,meta:a},i:"\\$[^01]|#[^0-9a-fA-F]",c:[e.CLCM,e.C("/\\*","\\*/",{c:["self"]}),{cN:"meta",b:'@[a-z]\\w*(?:\\:"[^"]*")?'}].concat(n)}})),e.registerLanguage("clean",(function(e){return{aliases:["clean","icl","dcl"],k:{keyword:"if let in with where case of class instance otherwise implementation definition system module from import qualified as special code inline foreign export ccall stdcall generic derive infix infixl infixr",literal:"True False"},c:[e.CLCM,e.CBCM,e.ASM,e.QSM,e.CNM,{b:"->|<-[|:]?|::|#!?|>>=|\\{\\||\\|\\}|:==|=:|\\.\\.|<>|`"}]}})),e.registerLanguage("clojure",(function(e){var t={"builtin-name":"def defonce cond apply if-not if-let if not not= = < > <= >= == + / * - rem quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last drop-while while intern condp case reduced cycle split-at split-with repeat replicate iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter monitor-exit defmacro defn defn- macroexpand macroexpand-1 for dosync and or when when-not when-let comp juxt partial sequence memoize constantly complement identity assert peek pop doto proxy defstruct first rest cons defprotocol cast coll deftype defrecord last butlast sigs reify second ffirst fnext nfirst nnext defmulti defmethod meta with-meta ns in-ns create-ns import refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! assoc! dissoc! pop! disj! use class type num float double short byte boolean bigint biginteger bigdec print-method print-dup throw-if printf format load compile get-in update-in pr pr-on newline flush read slurp read-line subvec with-open memfn time re-find re-groups rand-int rand mod locking assert-valid-fdecl alias resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! new next conj set! to-array future future-call into-array aset gen-class reduce map filter find empty hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"},r="a-zA-Z_\\-!.?+*=<>&#'",a="["+r+"]["+r+"0-9/;:]*",i="[-+]?\\d+(\\.\\d+)?",n={b:a,r:0},o={cN:"number",b:i,r:0},s=e.inherit(e.QSM,{i:null}),l=e.C(";","$",{r:0}),c={cN:"literal",b:/\b(true|false|nil)\b/},d={b:"[\\[\\{]",e:"[\\]\\}]"},p={cN:"comment",b:"\\^"+a},m=e.C("\\^\\{","\\}"),u={cN:"symbol",b:"[:]{1,2}"+a},b={b:"\\(",e:"\\)"},g={eW:!0,r:0},f={k:t,l:a,cN:"name",b:a,starts:g},_=[b,s,p,m,l,u,d,o,c,n];return b.c=[e.C("comment",""),f,g],g.c=_,d.c=_,m.c=[d],{aliases:["clj"],i:/\S/,c:[b,s,p,m,l,u,d,o,c]}})),e.registerLanguage("clojure-repl",(function(e){return{c:[{cN:"meta",b:/^([\w.-]+|\s*#_)=>/,starts:{e:/$/,sL:"clojure"}}]}})),e.registerLanguage("cmake",(function(e){return{aliases:["cmake.in"],cI:!0,k:{keyword:"add_custom_command add_custom_target add_definitions add_dependencies add_executable add_library add_subdirectory add_test aux_source_directory break build_command cmake_minimum_required cmake_policy configure_file create_test_sourcelist define_property else elseif enable_language enable_testing endforeach endfunction endif endmacro endwhile execute_process export find_file find_library find_package find_path find_program fltk_wrap_ui foreach function get_cmake_property get_directory_property get_filename_component get_property get_source_file_property get_target_property get_test_property if include include_directories include_external_msproject include_regular_expression install link_directories load_cache load_command macro mark_as_advanced message option output_required_files project qt_wrap_cpp qt_wrap_ui remove_definitions return separate_arguments set set_directory_properties set_property set_source_files_properties set_target_properties set_tests_properties site_name source_group string target_link_libraries try_compile try_run unset variable_watch while build_name exec_program export_library_dependencies install_files install_programs install_targets link_libraries make_directory remove subdir_depends subdirs use_mangled_mesa utility_source variable_requires write_file qt5_use_modules qt5_use_package qt5_wrap_cpp on off true false and or equal less greater strless strgreater strequal matches"},c:[{cN:"variable",b:"\\${",e:"}"},e.HCM,e.QSM,e.NM]}})),e.registerLanguage("coffeescript",(function(e){var t={keyword:"in if for while finally new do return else break catch instanceof throw try this switch continue typeof delete debugger super yield import export from as default await then unless until loop of by when and or is isnt not",literal:"true false null undefined yes no on off",built_in:"npm require console print module global window document"},r="[A-Za-z$_][0-9A-Za-z$_]*",a={cN:"subst",b:/#\{/,e:/}/,k:t},i=[e.BNM,e.inherit(e.CNM,{starts:{e:"(\\s*/)?",r:0}}),{cN:"string",v:[{b:/'''/,e:/'''/,c:[e.BE]},{b:/'/,e:/'/,c:[e.BE]},{b:/"""/,e:/"""/,c:[e.BE,a]},{b:/"/,e:/"/,c:[e.BE,a]}]},{cN:"regexp",v:[{b:"///",e:"///",c:[a,e.HCM]},{b:"//[gim]*",r:0},{b:/\/(?![ *])(\\\/|.)*?\/[gim]*(?=\W|$)/}]},{b:"@"+r},{sL:"javascript",eB:!0,eE:!0,v:[{b:"```",e:"```"},{b:"`",e:"`"}]}];a.c=i;var n=e.inherit(e.TM,{b:r}),o="(\\(.*\\))?\\s*\\B[-=]>",s={cN:"params",b:"\\([^\\(]",rB:!0,c:[{b:/\(/,e:/\)/,k:t,c:["self"].concat(i)}]};return{aliases:["coffee","cson","iced"],k:t,i:/\/\*/,c:i.concat([e.C("###","###"),e.HCM,{cN:"function",b:"^\\s*"+r+"\\s*=\\s*"+o,e:"[-=]>",rB:!0,c:[n,s]},{b:/[:\(,=]\s*/,r:0,c:[{cN:"function",b:o,e:"[-=]>",rB:!0,c:[s]}]},{cN:"class",bK:"class",e:"$",i:/[:="\[\]]/,c:[{bK:"extends",eW:!0,i:/[:="\[\]]/,c:[n]},n]},{b:r+":",e:":",rB:!0,rE:!0,r:0}])}})),e.registerLanguage("coq",(function(e){return{k:{keyword:"_ as at cofix else end exists exists2 fix for forall fun if IF in let match mod Prop return Set then Type using where with Abort About Add Admit Admitted All Arguments Assumptions Axiom Back BackTo Backtrack Bind Blacklist Canonical Cd Check Class Classes Close Coercion Coercions CoFixpoint CoInductive Collection Combined Compute Conjecture Conjectures Constant constr Constraint Constructors Context Corollary CreateHintDb Cut Declare Defined Definition Delimit Dependencies DependentDerive Drop eauto End Equality Eval Example Existential Existentials Existing Export exporting Extern Extract Extraction Fact Field Fields File Fixpoint Focus for From Function Functional Generalizable Global Goal Grab Grammar Graph Guarded Heap Hint HintDb Hints Hypotheses Hypothesis ident Identity If Immediate Implicit Import Include Inductive Infix Info Initial Inline Inspect Instance Instances Intro Intros Inversion Inversion_clear Language Left Lemma Let Libraries Library Load LoadPath Local Locate Ltac ML Mode Module Modules Monomorphic Morphism Next NoInline Notation Obligation Obligations Opaque Open Optimize Options Parameter Parameters Parametric Path Paths pattern Polymorphic Preterm Print Printing Program Projections Proof Proposition Pwd Qed Quit Rec Record Recursive Redirect Relation Remark Remove Require Reserved Reset Resolve Restart Rewrite Right Ring Rings Save Scheme Scope Scopes Script Search SearchAbout SearchHead SearchPattern SearchRewrite Section Separate Set Setoid Show Solve Sorted Step Strategies Strategy Structure SubClass Table Tables Tactic Term Test Theorem Time Timeout Transparent Type Typeclasses Types Undelimit Undo Unfocus Unfocused Unfold Universe Universes Unset Unshelve using Variable Variables Variant Verbose Visibility where with",built_in:"abstract absurd admit after apply as assert assumption at auto autorewrite autounfold before bottom btauto by case case_eq cbn cbv change classical_left classical_right clear clearbody cofix compare compute congruence constr_eq constructor contradict contradiction cut cutrewrite cycle decide decompose dependent destruct destruction dintuition discriminate discrR do double dtauto eapply eassumption eauto ecase econstructor edestruct ediscriminate eelim eexact eexists einduction einjection eleft elim elimtype enough equality erewrite eright esimplify_eq esplit evar exact exactly_once exfalso exists f_equal fail field field_simplify field_simplify_eq first firstorder fix fold fourier functional generalize generalizing gfail give_up has_evar hnf idtac in induction injection instantiate intro intro_pattern intros intuition inversion inversion_clear is_evar is_var lapply lazy left lia lra move native_compute nia nsatz omega once pattern pose progress proof psatz quote record red refine reflexivity remember rename repeat replace revert revgoals rewrite rewrite_strat right ring ring_simplify rtauto set setoid_reflexivity setoid_replace setoid_rewrite setoid_symmetry setoid_transitivity shelve shelve_unifiable simpl simple simplify_eq solve specialize split split_Rabs split_Rmult stepl stepr subst sum swap symmetry tactic tauto time timeout top transitivity trivial try tryif unfold unify until using vm_compute with"},c:[e.QSM,e.C("\\(\\*","\\*\\)"),e.CNM,{cN:"type",eB:!0,b:"\\|\\s*",e:"\\w+"},{b:/[-=]>/}]}})),e.registerLanguage("cos",(function(e){var t={cN:"string",v:[{b:'"',e:'"',c:[{b:'""',r:0}]}]},r={cN:"number",b:"\\b(\\d+(\\.\\d*)?|\\.\\d+)",r:0},a="property parameter class classmethod clientmethod extends as break catch close continue do d|0 else elseif for goto halt hang h|0 if job j|0 kill k|0 lock l|0 merge new open quit q|0 read r|0 return set s|0 tcommit throw trollback try tstart use view while write w|0 xecute x|0 zkill znspace zn ztrap zwrite zw zzdump zzwrite print zbreak zinsert zload zprint zremove zsave zzprint mv mvcall mvcrt mvdim mvprint zquit zsync ascii";return{cI:!0,aliases:["cos","cls"],k:a,c:[r,t,e.CLCM,e.CBCM,{cN:"comment",b:/;/,e:"$",r:0},{cN:"built_in",b:/(?:\$\$?|\.\.)\^?[a-zA-Z]+/},{cN:"built_in",b:/\$\$\$[a-zA-Z]+/},{cN:"built_in",b:/%[a-z]+(?:\.[a-z]+)*/},{cN:"symbol",b:/\^%?[a-zA-Z][\w]*/},{cN:"keyword",b:/##class|##super|#define|#dim/},{b:/&sql\(/,e:/\)/,eB:!0,eE:!0,sL:"sql"},{b:/&(js|jscript|javascript)/,eB:!0,eE:!0,sL:"javascript"},{b:/&html<\s*\s*>/,sL:"xml"}]}})),e.registerLanguage("crmsh",(function(e){var t="primitive rsc_template",r="group clone ms master location colocation order fencing_topology rsc_ticket acl_target acl_group user role tag xml",a="property rsc_defaults op_defaults",i="params meta operations op rule attributes utilization",n="read write deny defined not_defined in_range date spec in ref reference attribute type xpath version and or lt gt tag lte gte eq ne \\",o="number string",s="Master Started Slave Stopped start promote demote stop monitor true false";return{aliases:["crm","pcmk"],cI:!0,k:{keyword:i+" "+n+" "+o,literal:s},c:[e.HCM,{bK:"node",starts:{e:"\\s*([\\w_-]+:)?",starts:{cN:"title",e:"\\s*[\\$\\w_][\\w_-]*"}}},{bK:t,starts:{cN:"title",e:"\\s*[\\$\\w_][\\w_-]*",starts:{e:"\\s*@?[\\w_][\\w_\\.:-]*"}}},{b:"\\b("+r.split(" ").join("|")+")\\s+",k:r,starts:{cN:"title",e:"[\\$\\w_][\\w_-]*"}},{bK:a,starts:{cN:"title",e:"\\s*([\\w_-]+:)?"}},e.QSM,{cN:"meta",b:"(ocf|systemd|service|lsb):[\\w_:-]+",r:0},{cN:"number",b:"\\b\\d+(\\.\\d+)?(ms|s|h|m)?",r:0},{cN:"literal",b:"[-]?(infinity|inf)",r:0},{cN:"attr",b:/([A-Za-z\$_\#][\w_-]+)=/,r:0},{cN:"tag",b:"",r:0}]}})),e.registerLanguage("crystal",(function(e){function t(e,t){var r=[{b:e,e:t}];return r[0].c=r,r}var r="(_[uif](8|16|32|64))?",a="[a-zA-Z_]\\w*[!?=]?",i="!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\][=?]?",o={keyword:"abstract alias as as? asm begin break case class def do else elsif end ensure enum extend for fun if include instance_sizeof is_a? lib macro module next nil? of out pointerof private protected rescue responds_to? return require select self sizeof struct super then type typeof union uninitialized unless until when while with yield __DIR__ __END_LINE__ __FILE__ __LINE__",literal:"false nil true"},s={cN:"subst",b:"#{",e:"}",k:o},l={cN:"template-variable",v:[{b:"\\{\\{",e:"\\}\\}"},{b:"\\{%",e:"%\\}"}],k:o},c={cN:"string",c:[e.BE,s],v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/`/,e:/`/},{b:"%w?\\(",e:"\\)",c:t("\\(","\\)")},{b:"%w?\\[",e:"\\]",c:t("\\[","\\]")},{b:"%w?{",e:"}",c:t("{","}")},{b:"%w?<",e:">",c:t("<",">")},{b:"%w?/",e:"/"},{b:"%w?%",e:"%"},{b:"%w?-",e:"-"},{b:"%w?\\|",e:"\\|"},{b:/<<-\w+$/,e:/^\s*\w+$/}],r:0},d={cN:"string",v:[{b:"%q\\(",e:"\\)",c:t("\\(","\\)")},{b:"%q\\[",e:"\\]",c:t("\\[","\\]")},{b:"%q{",e:"}",c:t("{","}")},{b:"%q<",e:">",c:t("<",">")},{b:"%q/",e:"/"},{b:"%q%",e:"%"},{b:"%q-",e:"-"},{b:"%q\\|",e:"\\|"},{b:/<<-'\w+'$/,e:/^\s*\w+$/}],r:0},p={b:"("+i+")\\s*",c:[{cN:"regexp",c:[e.BE,s],v:[{b:"//[a-z]*",r:0},{b:"/",e:"/[a-z]*"},{b:"%r\\(",e:"\\)",c:t("\\(","\\)")},{b:"%r\\[",e:"\\]",c:t("\\[","\\]")},{b:"%r{",e:"}",c:t("{","}")},{b:"%r<",e:">",c:t("<",">")},{b:"%r/",e:"/"},{b:"%r%",e:"%"},{b:"%r-",e:"-"},{b:"%r\\|",e:"\\|"}]}],r:0},m={cN:"regexp",c:[e.BE,s],v:[{b:"%r\\(",e:"\\)",c:t("\\(","\\)")},{b:"%r\\[",e:"\\]",c:t("\\[","\\]")},{b:"%r{",e:"}",c:t("{","}")},{b:"%r<",e:">",c:t("<",">")},{b:"%r/",e:"/"},{b:"%r%",e:"%"},{b:"%r-",e:"-"},{b:"%r\\|",e:"\\|"}],r:0},u={cN:"meta",b:"@\\[",e:"\\]",c:[e.inherit(e.QSM,{cN:"meta-string"})]},b=[l,c,d,p,m,u,e.HCM,{cN:"class",bK:"class module struct",e:"$|;",i:/=/,c:[e.HCM,e.inherit(e.TM,{b:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{b:"<"}]},{cN:"class",bK:"lib enum union",e:"$|;",i:/=/,c:[e.HCM,e.inherit(e.TM,{b:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"})],r:10},{cN:"function",bK:"def",e:/\B\b/,c:[e.inherit(e.TM,{b:n,endsParent:!0})]},{cN:"function",bK:"fun macro",e:/\B\b/,c:[e.inherit(e.TM,{b:n,endsParent:!0})],r:5},{cN:"symbol",b:e.UIR+"(\\!|\\?)?:",r:0},{cN:"symbol",b:":",c:[c,{b:n}],r:0},{cN:"number",v:[{b:"\\b0b([01_]*[01])"+r},{b:"\\b0o([0-7_]*[0-7])"+r},{b:"\\b0x([A-Fa-f0-9_]*[A-Fa-f0-9])"+r},{b:"\\b(([0-9][0-9_]*[0-9]|[0-9])(\\.[0-9_]*[0-9])?([eE][+-]?[0-9_]*[0-9])?)"+r}],r:0}];return s.c=b,l.c=b.slice(1),{aliases:["cr"],l:a,k:o,c:b}})),e.registerLanguage("cs",(function(e){var t={keyword:"abstract as base bool break byte case catch char checked const continue decimal default delegate do double enum event explicit extern finally fixed float for foreach goto if implicit in int interface internal is lock long nameof object operator out override params private protected public readonly ref sbyte sealed short sizeof stackalloc static string struct switch this try typeof uint ulong unchecked unsafe ushort using virtual void volatile while add alias ascending async await by descending dynamic equals from get global group into join let on orderby partial remove select set value var where yield",literal:"null false true"},r={cN:"string",b:'@"',e:'"',c:[{b:'""'}]},a=e.inherit(r,{i:/\n/}),i={cN:"subst",b:"{",e:"}",k:t},n=e.inherit(i,{i:/\n/}),o={cN:"string",b:/\$"/,e:'"',i:/\n/,c:[{b:"{{"},{b:"}}"},e.BE,n]},s={cN:"string",b:/\$@"/,e:'"',c:[{b:"{{"},{b:"}}"},{b:'""'},i]},l=e.inherit(s,{i:/\n/,c:[{b:"{{"},{b:"}}"},{b:'""'},n]});i.c=[s,o,r,e.ASM,e.QSM,e.CNM,e.CBCM],n.c=[l,o,a,e.ASM,e.QSM,e.CNM,e.inherit(e.CBCM,{i:/\n/})];var c={v:[s,o,r,e.ASM,e.QSM]},d=e.IR+"(<"+e.IR+"(\\s*,\\s*"+e.IR+")*>)?(\\[\\])?";return{aliases:["csharp"],k:t,i:/::/,c:[e.C("///","$",{rB:!0,c:[{cN:"doctag",v:[{b:"///",r:0},{b:"\x3c!--|--\x3e"},{b:""}]}]}),e.CLCM,e.CBCM,{cN:"meta",b:"#",e:"$",k:{"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum"}},c,e.CNM,{bK:"class interface",e:/[{;=]/,i:/[^\s:]/,c:[e.TM,e.CLCM,e.CBCM]},{bK:"namespace",e:/[{;=]/,i:/[^\s:]/,c:[e.inherit(e.TM,{b:"[a-zA-Z](\\.?\\w)*"}),e.CLCM,e.CBCM]},{cN:"meta",b:"^\\s*\\[",eB:!0,e:"\\]",eE:!0,c:[{cN:"meta-string",b:/"/,e:/"/}]},{bK:"new return throw await else",r:0},{cN:"function",b:"("+d+"\\s+)+"+e.IR+"\\s*\\(",rB:!0,e:/[{;=]/,eE:!0,k:t,c:[{b:e.IR+"\\s*\\(",rB:!0,c:[e.TM],r:0},{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:t,r:0,c:[c,e.CNM,e.CBCM]},e.CLCM,e.CBCM]}]}})),e.registerLanguage("csp",(function(e){return{cI:!1,l:"[a-zA-Z][a-zA-Z0-9_-]*",k:{keyword:"base-uri child-src connect-src default-src font-src form-action frame-ancestors frame-src img-src media-src object-src plugin-types report-uri sandbox script-src style-src"},c:[{cN:"string",b:"'",e:"'"},{cN:"attribute",b:"^Content",e:":",eE:!0}]}})),e.registerLanguage("css",(function(e){var t="[a-zA-Z-][a-zA-Z0-9_-]*",r={b:/[A-Z\_\.\-]+\s*:/,rB:!0,e:";",eW:!0,c:[{cN:"attribute",b:/\S/,e:":",eE:!0,starts:{eW:!0,eE:!0,c:[{b:/[\w-]+\(/,rB:!0,c:[{cN:"built_in",b:/[\w-]+/},{b:/\(/,e:/\)/,c:[e.ASM,e.QSM]}]},e.CSSNM,e.QSM,e.ASM,e.CBCM,{cN:"number",b:"#[0-9A-Fa-f]+"},{cN:"meta",b:"!important"}]}}]};return{cI:!0,i:/[=\/|'\$]/,c:[e.CBCM,{cN:"selector-id",b:/#[A-Za-z0-9_-]+/},{cN:"selector-class",b:/\.[A-Za-z0-9_-]+/},{cN:"selector-attr",b:/\[/,e:/\]/,i:"$"},{cN:"selector-pseudo",b:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{b:"@(font-face|page)",l:"[a-z-]+",k:"font-face page"},{b:"@",e:"[{;]",i:/:/,c:[{cN:"keyword",b:/\w+/},{b:/\s/,eW:!0,eE:!0,r:0,c:[e.ASM,e.QSM,e.CSSNM]}]},{cN:"selector-tag",b:t,r:0},{b:"{",e:"}",i:/\S/,c:[e.CBCM,r]}]}})),e.registerLanguage("d",(function(e){var t={keyword:"abstract alias align asm assert auto body break byte case cast catch class const continue debug default delete deprecated do else enum export extern final finally for foreach foreach_reverse|10 goto if immutable import in inout int interface invariant is lazy macro mixin module new nothrow out override package pragma private protected public pure ref return scope shared static struct super switch synchronized template this throw try typedef typeid typeof union unittest version void volatile while with __FILE__ __LINE__ __gshared|10 __thread __traits __DATE__ __EOF__ __TIME__ __TIMESTAMP__ __VENDOR__ __VERSION__",built_in:"bool cdouble cent cfloat char creal dchar delegate double dstring float function idouble ifloat ireal long real short string ubyte ucent uint ulong ushort wchar wstring",literal:"false null true"},r="(0|[1-9][\\d_]*)",a="(0|[1-9][\\d_]*|\\d[\\d_]*|[\\d_]+?\\d)",i="0[bB][01_]+",n="([\\da-fA-F][\\da-fA-F_]*|_[\\da-fA-F][\\da-fA-F_]*)",o="0[xX]"+n,s="([eE][+-]?"+a+")",l="("+a+"(\\.\\d*|"+s+")|\\d+\\."+a+a+"|\\."+r+s+"?)",c="(0[xX]("+n+"\\."+n+"|\\.?"+n+")[pP][+-]?"+a+")",d="("+r+"|"+i+"|"+o+")",p="("+c+"|"+l+")",m="\\\\(['\"\\?\\\\abfnrtv]|u[\\dA-Fa-f]{4}|[0-7]{1,3}|x[\\dA-Fa-f]{2}|U[\\dA-Fa-f]{8})|&[a-zA-Z\\d]{2,};",u={cN:"number",b:"\\b"+d+"(L|u|U|Lu|LU|uL|UL)?",r:0},b={cN:"number",b:"\\b("+p+"([fF]|L|i|[fF]i|Li)?|"+d+"(i|[fF]i|Li))",r:0},g={cN:"string",b:"'("+m+"|.)",e:"'",i:"."},f={b:m,r:0},_={cN:"string",b:'"',c:[f],e:'"[cwd]?'},h={cN:"string",b:'[rq]"',e:'"[cwd]?',r:5},v={cN:"string",b:"`",e:"`[cwd]?"},y={cN:"string",b:'x"[\\da-fA-F\\s\\n\\r]*"[cwd]?',r:10},S={cN:"string",b:'q"\\{',e:'\\}"'},C={cN:"meta",b:"^#!",e:"$",r:5},x={cN:"meta",b:"#(line)",e:"$",r:5},E={cN:"keyword",b:"@[a-zA-Z_][a-zA-Z_\\d]*"},N=e.C("\\/\\+","\\+\\/",{c:["self"],r:10});return{l:e.UIR,k:t,c:[e.CLCM,e.CBCM,N,y,_,h,v,S,b,u,g,C,x,E]}})),e.registerLanguage("markdown",(function(e){return{aliases:["md","mkdown","mkd"],c:[{cN:"section",v:[{b:"^#{1,6}",e:"$"},{b:"^.+?\\n[=-]{2,}$"}]},{b:"<",e:">",sL:"xml",r:0},{cN:"bullet",b:"^([*+-]|(\\d+\\.))\\s+"},{cN:"strong",b:"[*_]{2}.+?[*_]{2}"},{cN:"emphasis",v:[{b:"\\*.+?\\*"},{b:"_.+?_",r:0}]},{cN:"quote",b:"^>\\s+",e:"$"},{cN:"code",v:[{b:"^```w*s*$",e:"^```s*$"},{b:"`.+?`"},{b:"^( {4}|\t)",e:"$",r:0}]},{b:"^[-\\*]{3,}",e:"$"},{b:"\\[.+?\\][\\(\\[].*?[\\)\\]]",rB:!0,c:[{cN:"string",b:"\\[",e:"\\]",eB:!0,rE:!0,r:0},{cN:"link",b:"\\]\\(",e:"\\)",eB:!0,eE:!0},{cN:"symbol",b:"\\]\\[",e:"\\]",eB:!0,eE:!0}],r:10},{b:/^\[[^\n]+\]:/,rB:!0,c:[{cN:"symbol",b:/\[/,e:/\]/,eB:!0,eE:!0},{cN:"link",b:/:\s*/,e:/$/,eB:!0}]}]}})),e.registerLanguage("dart",(function(e){var t={cN:"subst",b:"\\$\\{",e:"}",k:"true false null this is new super"},r={cN:"string",v:[{b:"r'''",e:"'''"},{b:'r"""',e:'"""'},{b:"r'",e:"'",i:"\\n"},{b:'r"',e:'"',i:"\\n"},{b:"'''",e:"'''",c:[e.BE,t]},{b:'"""',e:'"""',c:[e.BE,t]},{b:"'",e:"'",i:"\\n",c:[e.BE,t]},{b:'"',e:'"',i:"\\n",c:[e.BE,t]}]};t.c=[e.CNM,r];var a={keyword:"assert async await break case catch class const continue default do else enum extends false final finally for if in is new null rethrow return super switch sync this throw true try var void while with yield abstract as dynamic export external factory get implements import library operator part set static typedef",built_in:"print Comparable DateTime Duration Function Iterable Iterator List Map Match Null Object Pattern RegExp Set Stopwatch String StringBuffer StringSink Symbol Type Uri bool double int num document window querySelector querySelectorAll Element ElementList"};return{k:a,c:[r,e.C("/\\*\\*","\\*/",{sL:"markdown"}),e.C("///","$",{sL:"markdown"}),e.CLCM,e.CBCM,{cN:"class",bK:"class interface",e:"{",eE:!0,c:[{bK:"extends implements"},e.UTM]},e.CNM,{cN:"meta",b:"@[A-Za-z]+"},{b:"=>"}]}})),e.registerLanguage("delphi",(function(e){var t="exports register file shl array record property for mod while set ally label uses raise not stored class safecall var interface or private static exit index inherited to else stdcall override shr asm far resourcestring finalization packed virtual out and protected library do xorwrite goto near function end div overload object unit begin string on inline repeat until destructor write message program with read initialization except default nil if case cdecl in downto threadvar of try pascal const external constructor type public then implementation finally published procedure absolute reintroduce operator as is abstract alias assembler bitpacked break continue cppdecl cvar enumerator experimental platform deprecated unimplemented dynamic export far16 forward generic helper implements interrupt iochecks local name nodefault noreturn nostackframe oldfpccall otherwise saveregisters softfloat specialize strict unaligned varargs ",r=[e.CLCM,e.C(/\{/,/\}/,{r:0}),e.C(/\(\*/,/\*\)/,{r:10})],a={cN:"meta",v:[{b:/\{\$/,e:/\}/},{b:/\(\*\$/,e:/\*\)/}]},i={cN:"string",b:/'/,e:/'/,c:[{b:/''/}]},n={cN:"string",b:/(#\d+)+/},o={b:e.IR+"\\s*=\\s*class\\s*\\(",rB:!0,c:[e.TM]},s={cN:"function",bK:"function constructor destructor procedure",e:/[:;]/,k:"function constructor|10 destructor|10 procedure|10",c:[e.TM,{cN:"params",b:/\(/,e:/\)/,k:t,c:[i,n,a].concat(r)},a].concat(r)};return{aliases:["dpr","dfm","pas","pascal","freepascal","lazarus","lpr","lfm"],cI:!0,k:t,i:/"|\$[G-Zg-z]|\/\*|<\/|\|/,c:[i,n,e.NM,o,s,a].concat(r)}})),e.registerLanguage("diff",(function(e){return{aliases:["patch"],c:[{cN:"meta",r:10,v:[{b:/^@@ +\-\d+,\d+ +\+\d+,\d+ +@@$/},{b:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{b:/^\-\-\- +\d+,\d+ +\-\-\-\-$/}]},{cN:"comment",v:[{b:/Index: /,e:/$/},{b:/={3,}/,e:/$/},{b:/^\-{3}/,e:/$/},{b:/^\*{3} /,e:/$/},{b:/^\+{3}/,e:/$/},{b:/\*{5}/,e:/\*{5}$/}]},{cN:"addition",b:"^\\+",e:"$"},{cN:"deletion",b:"^\\-",e:"$"},{cN:"addition",b:"^\\!",e:"$"}]}})),e.registerLanguage("django",(function(e){var t={b:/\|[A-Za-z]+:?/,k:{name:"truncatewords removetags linebreaksbr yesno get_digit timesince random striptags filesizeformat escape linebreaks length_is ljust rjust cut urlize fix_ampersands title floatformat capfirst pprint divisibleby add make_list unordered_list urlencode timeuntil urlizetrunc wordcount stringformat linenumbers slice date dictsort dictsortreversed default_if_none pluralize lower join center default truncatewords_html upper length phone2numeric wordwrap time addslashes slugify first escapejs force_escape iriencode last safe safeseq truncatechars localize unlocalize localtime utc timezone"},c:[e.QSM,e.ASM]};return{aliases:["jinja"],cI:!0,sL:"xml",c:[e.C(/\{%\s*comment\s*%}/,/\{%\s*endcomment\s*%}/),e.C(/\{#/,/#}/),{cN:"template-tag",b:/\{%/,e:/%}/,c:[{cN:"name",b:/\w+/,k:{name:"comment endcomment load templatetag ifchanged endifchanged if endif firstof for endfor ifnotequal endifnotequal widthratio extends include spaceless endspaceless regroup ifequal endifequal ssi now with cycle url filter endfilter debug block endblock else autoescape endautoescape csrf_token empty elif endwith static trans blocktrans endblocktrans get_static_prefix get_media_prefix plural get_current_language language get_available_languages get_current_language_bidi get_language_info get_language_info_list localize endlocalize localtime endlocaltime timezone endtimezone get_current_timezone verbatim"},starts:{eW:!0,k:"in by as",c:[t],r:0}}]},{cN:"template-variable",b:/\{\{/,e:/}}/,c:[t]}]}})),e.registerLanguage("dns",(function(e){return{aliases:["bind","zone"],k:{keyword:"IN A AAAA AFSDB APL CAA CDNSKEY CDS CERT CNAME DHCID DLV DNAME DNSKEY DS HIP IPSECKEY KEY KX LOC MX NAPTR NS NSEC NSEC3 NSEC3PARAM PTR RRSIG RP SIG SOA SRV SSHFP TA TKEY TLSA TSIG TXT"},c:[e.C(";","$",{r:0}),{cN:"meta",b:/^\$(TTL|GENERATE|INCLUDE|ORIGIN)\b/},{cN:"number",b:"((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))\\b"},{cN:"number",b:"((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\b"},e.inherit(e.NM,{b:/\b\d+[dhwm]?/})]}})),e.registerLanguage("dockerfile",(function(e){return{aliases:["docker"],cI:!0,k:"from maintainer expose env arg user onbuild stopsignal",c:[e.HCM,e.ASM,e.QSM,e.NM,{bK:"run cmd entrypoint volume add copy workdir label healthcheck shell",starts:{e:/[^\\]\n/,sL:"bash"}}],i:"",i:"\\n"}]},t,e.CLCM,e.CBCM]},i={cN:"variable",b:"\\&[a-z\\d_]*\\b"},n={cN:"meta-keyword",b:"/[a-z][a-z\\d-]*/"},o={cN:"symbol",b:"^\\s*[a-zA-Z_][a-zA-Z\\d_]*:"},s={cN:"params",b:"<",e:">",c:[r,i]},l={cN:"class",b:/[a-zA-Z_][a-zA-Z\d_@]*\s{/,e:/[{;=]/,rB:!0,eE:!0},c={cN:"class",b:"/\\s*{",e:"};",r:10,c:[i,n,o,l,s,e.CLCM,e.CBCM,r,t]};return{k:"",c:[c,i,n,o,l,s,e.CLCM,e.CBCM,r,t,a,{b:e.IR+"::",k:""}]}})),e.registerLanguage("dust",(function(e){var t="if eq ne lt lte gt gte select default math sep";return{aliases:["dst"],cI:!0,sL:"xml",c:[{cN:"template-tag",b:/\{[#\/]/,e:/\}/,i:/;/,c:[{cN:"name",b:/[a-zA-Z\.-]+/,starts:{eW:!0,r:0,c:[e.QSM]}}]},{cN:"template-variable",b:/\{/,e:/\}/,i:/;/,k:t}]}})),e.registerLanguage("ebnf",(function(e){var t=e.C(/\(\*/,/\*\)/),r={cN:"attribute",b:/^[ ]*[a-zA-Z][a-zA-Z-]*([\s-]+[a-zA-Z][a-zA-Z]*)*/},a={cN:"meta",b:/\?.*\?/},i={b:/=/,e:/;/,c:[t,a,e.ASM,e.QSM]};return{i:/\S/,c:[t,r,i]}})),e.registerLanguage("elixir",(function(e){var t="[a-zA-Z_][a-zA-Z0-9_]*(\\!|\\?)?",r="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",a="and false then defined module in return redo retry end for true self when next until do begin unless nil break not case cond alias while ensure or include use alias fn quote",i={cN:"subst",b:"#\\{",e:"}",l:t,k:a},n={cN:"string",c:[e.BE,i],v:[{b:/'/,e:/'/},{b:/"/,e:/"/}]},o={cN:"function",bK:"def defp defmacro",e:/\B\b/,c:[e.inherit(e.TM,{b:t,endsParent:!0})]},s=e.inherit(o,{cN:"class",bK:"defimpl defmodule defprotocol defrecord",e:/\bdo\b|$|;/}),l=[n,e.HCM,s,o,{cN:"symbol",b:":(?!\\s)",c:[n,{b:r}],r:0},{cN:"symbol",b:t+":",r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{cN:"variable",b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{b:"->"},{b:"("+e.RSR+")\\s*",c:[e.HCM,{cN:"regexp",i:"\\n",c:[e.BE,i],v:[{b:"/",e:"/[a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}],r:0}];return i.c=l,{l:t,k:a,c:l}})),e.registerLanguage("elm",(function(e){var t={v:[e.C("--","$"),e.C("{-","-}",{c:["self"]})]},r={cN:"type",b:"\\b[A-Z][\\w']*",r:0},a={b:"\\(",e:"\\)",i:'"',c:[{cN:"type",b:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},t]},i={b:"{",e:"}",c:a.c};return{k:"let in if then else case of where module import exposing type alias as infix infixl infixr port effect command subscription",c:[{bK:"port effect module",e:"exposing",k:"port effect module where command subscription exposing",c:[a,t],i:"\\W\\.|;"},{b:"import",e:"$",k:"import as exposing",c:[a,t],i:"\\W\\.|;"},{b:"type",e:"$",k:"type alias",c:[r,a,i,t]},{bK:"infix infixl infixr",e:"$",c:[e.CNM,t]},{b:"port",e:"$",k:"port",c:[t]},e.QSM,e.CNM,r,e.inherit(e.TM,{b:"^[_a-z][\\w']*"}),t,{b:"->|<-"}],i:/;/}})),e.registerLanguage("ruby",(function(e){var t="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",r={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},a={cN:"doctag",b:"@[A-Za-z]+"},i={b:"#<",e:">"},n=[e.C("#","$",{c:[a]}),e.C("^\\=begin","^\\=end",{c:[a],r:10}),e.C("^__END__","\\n$")],o={cN:"subst",b:"#\\{",e:"}",k:r},s={cN:"string",c:[e.BE,o],v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/`/,e:/`/},{b:"%[qQwWx]?\\(",e:"\\)"},{b:"%[qQwWx]?\\[",e:"\\]"},{b:"%[qQwWx]?{",e:"}"},{b:"%[qQwWx]?<",e:">"},{b:"%[qQwWx]?/",e:"/"},{b:"%[qQwWx]?%",e:"%"},{b:"%[qQwWx]?-",e:"-"},{b:"%[qQwWx]?\\|",e:"\\|"},{b:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{b:/<<(-?)\w+$/,e:/^\s*\w+$/}]},l={cN:"params",b:"\\(",e:"\\)",endsParent:!0,k:r},c=[s,i,{cN:"class",bK:"class module",e:"$|;",i:/=/,c:[e.inherit(e.TM,{b:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{b:"<\\s*",c:[{b:"("+e.IR+"::)?"+e.IR}]}].concat(n)},{cN:"function",bK:"def",e:"$|;",c:[e.inherit(e.TM,{b:t}),l].concat(n)},{b:e.IR+"::"},{cN:"symbol",b:e.UIR+"(\\!|\\?)?:",r:0},{cN:"symbol",b:":(?!\\s)",c:[s,{b:t}],r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{cN:"params",b:/\|/,e:/\|/,k:r},{b:"("+e.RSR+"|unless)\\s*",k:"unless",c:[i,{cN:"regexp",c:[e.BE,o],i:/\n/,v:[{b:"/",e:"/[a-z]*"},{b:"%r{",e:"}[a-z]*"},{b:"%r\\(",e:"\\)[a-z]*"},{b:"%r!",e:"![a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}].concat(n),r:0}].concat(n);o.c=c,l.c=c;var d="[>?]>",p="[\\w#]+\\(\\w+\\):\\d+:\\d+>",m="(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>",u=[{b:/^\s*=>/,starts:{e:"$",c:c}},{cN:"meta",b:"^("+d+"|"+p+"|"+m+")",starts:{e:"$",c:c}}];return{aliases:["rb","gemspec","podspec","thor","irb"],k:r,i:/\/\*/,c:n.concat(u).concat(c)}})),e.registerLanguage("erb",(function(e){return{sL:"xml",c:[e.C("<%#","%>"),{b:"<%[%=-]?",e:"[%-]?%>",sL:"ruby",eB:!0,eE:!0}]}})),e.registerLanguage("erlang-repl",(function(e){return{k:{built_in:"spawn spawn_link self",keyword:"after and andalso|10 band begin bnot bor bsl bsr bxor case catch cond div end fun if let not of or orelse|10 query receive rem try when xor"},c:[{cN:"meta",b:"^[0-9]+> ",r:10},e.C("%","$"),{cN:"number",b:"\\b(\\d+#[a-fA-F0-9]+|\\d+(\\.\\d+)?([eE][-+]?\\d+)?)",r:0},e.ASM,e.QSM,{b:"\\?(::)?([A-Z]\\w*(::)?)+"},{b:"->"},{b:"ok"},{b:"!"},{b:"(\\b[a-z'][a-zA-Z0-9_']*:[a-z'][a-zA-Z0-9_']*)|(\\b[a-z'][a-zA-Z0-9_']*)",r:0},{b:"[A-Z][a-zA-Z0-9_']*",r:0}]}})),e.registerLanguage("erlang",(function(e){var t="[a-z'][a-zA-Z0-9_']*",r="("+t+":"+t+"|"+t+")",a={keyword:"after and andalso|10 band begin bnot bor bsl bzr bxor case catch cond div end fun if let not of orelse|10 query receive rem try when xor",literal:"false true"},i=e.C("%","$"),n={cN:"number",b:"\\b(\\d+#[a-fA-F0-9]+|\\d+(\\.\\d+)?([eE][-+]?\\d+)?)",r:0},o={b:"fun\\s+"+t+"/\\d+"},s={b:r+"\\(",e:"\\)",rB:!0,r:0,c:[{b:r,r:0},{b:"\\(",e:"\\)",eW:!0,rE:!0,r:0}]},l={b:"{",e:"}",r:0},c={b:"\\b_([A-Z][A-Za-z0-9_]*)?",r:0},d={b:"[A-Z][a-zA-Z0-9_]*",r:0},p={b:"#"+e.UIR,r:0,rB:!0,c:[{b:"#"+e.UIR,r:0},{b:"{",e:"}",r:0}]},m={bK:"fun receive if try case",e:"end",k:a};m.c=[i,o,e.inherit(e.ASM,{cN:""}),m,s,e.QSM,n,l,c,d,p];var u=[i,o,m,s,e.QSM,n,l,c,d,p];s.c[1].c=u,l.c=u,p.c[1].c=u;var b={cN:"params",b:"\\(",e:"\\)",c:u};return{aliases:["erl"],k:a,i:"(",rB:!0,i:"\\(|#|//|/\\*|\\\\|:|;",c:[b,e.inherit(e.TM,{b:t})],starts:{e:";|\\.",k:a,c:u}},i,{b:"^-",e:"\\.",r:0,eE:!0,rB:!0,l:"-"+e.IR,k:"-module -record -undef -export -ifdef -ifndef -author -copyright -doc -vsn -import -include -include_lib -compile -define -else -endif -file -behavior -behavior -spec",c:[b]},n,e.QSM,p,c,d,l,{b:/\.$/}]}})),e.registerLanguage("excel",(function(e){return{aliases:["xlsx","xls"],cI:!0,l:/[a-zA-Z][\w\.]*/,k:{built_in:"ABS ACCRINT ACCRINTM ACOS ACOSH ACOT ACOTH AGGREGATE ADDRESS AMORDEGRC AMORLINC AND ARABIC AREAS ASC ASIN ASINH ATAN ATAN2 ATANH AVEDEV AVERAGE AVERAGEA AVERAGEIF AVERAGEIFS BAHTTEXT BASE BESSELI BESSELJ BESSELK BESSELY BETADIST BETA.DIST BETAINV BETA.INV BIN2DEC BIN2HEX BIN2OCT BINOMDIST BINOM.DIST BINOM.DIST.RANGE BINOM.INV BITAND BITLSHIFT BITOR BITRSHIFT BITXOR CALL CEILING CEILING.MATH CEILING.PRECISE CELL CHAR CHIDIST CHIINV CHITEST CHISQ.DIST CHISQ.DIST.RT CHISQ.INV CHISQ.INV.RT CHISQ.TEST CHOOSE CLEAN CODE COLUMN COLUMNS COMBIN COMBINA COMPLEX CONCAT CONCATENATE CONFIDENCE CONFIDENCE.NORM CONFIDENCE.T CONVERT CORREL COS COSH COT COTH COUNT COUNTA COUNTBLANK COUNTIF COUNTIFS COUPDAYBS COUPDAYS COUPDAYSNC COUPNCD COUPNUM COUPPCD COVAR COVARIANCE.P COVARIANCE.S CRITBINOM CSC CSCH CUBEKPIMEMBER CUBEMEMBER CUBEMEMBERPROPERTY CUBERANKEDMEMBER CUBESET CUBESETCOUNT CUBEVALUE CUMIPMT CUMPRINC DATE DATEDIF DATEVALUE DAVERAGE DAY DAYS DAYS360 DB DBCS DCOUNT DCOUNTA DDB DEC2BIN DEC2HEX DEC2OCT DECIMAL DEGREES DELTA DEVSQ DGET DISC DMAX DMIN DOLLAR DOLLARDE DOLLARFR DPRODUCT DSTDEV DSTDEVP DSUM DURATION DVAR DVARP EDATE EFFECT ENCODEURL EOMONTH ERF ERF.PRECISE ERFC ERFC.PRECISE ERROR.TYPE EUROCONVERT EVEN EXACT EXP EXPON.DIST EXPONDIST FACT FACTDOUBLE FALSE|0 F.DIST FDIST F.DIST.RT FILTERXML FIND FINDB F.INV F.INV.RT FINV FISHER FISHERINV FIXED FLOOR FLOOR.MATH FLOOR.PRECISE FORECAST FORECAST.ETS FORECAST.ETS.CONFINT FORECAST.ETS.SEASONALITY FORECAST.ETS.STAT FORECAST.LINEAR FORMULATEXT FREQUENCY F.TEST FTEST FV FVSCHEDULE GAMMA GAMMA.DIST GAMMADIST GAMMA.INV GAMMAINV GAMMALN GAMMALN.PRECISE GAUSS GCD GEOMEAN GESTEP GETPIVOTDATA GROWTH HARMEAN HEX2BIN HEX2DEC HEX2OCT HLOOKUP HOUR HYPERLINK HYPGEOM.DIST HYPGEOMDIST IF|0 IFERROR IFNA IFS IMABS IMAGINARY IMARGUMENT IMCONJUGATE IMCOS IMCOSH IMCOT IMCSC IMCSCH IMDIV IMEXP IMLN IMLOG10 IMLOG2 IMPOWER IMPRODUCT IMREAL IMSEC IMSECH IMSIN IMSINH IMSQRT IMSUB IMSUM IMTAN INDEX INDIRECT INFO INT INTERCEPT INTRATE IPMT IRR ISBLANK ISERR ISERROR ISEVEN ISFORMULA ISLOGICAL ISNA ISNONTEXT ISNUMBER ISODD ISREF ISTEXT ISO.CEILING ISOWEEKNUM ISPMT JIS KURT LARGE LCM LEFT LEFTB LEN LENB LINEST LN LOG LOG10 LOGEST LOGINV LOGNORM.DIST LOGNORMDIST LOGNORM.INV LOOKUP LOWER MATCH MAX MAXA MAXIFS MDETERM MDURATION MEDIAN MID MIDBs MIN MINIFS MINA MINUTE MINVERSE MIRR MMULT MOD MODE MODE.MULT MODE.SNGL MONTH MROUND MULTINOMIAL MUNIT N NA NEGBINOM.DIST NEGBINOMDIST NETWORKDAYS NETWORKDAYS.INTL NOMINAL NORM.DIST NORMDIST NORMINV NORM.INV NORM.S.DIST NORMSDIST NORM.S.INV NORMSINV NOT NOW NPER NPV NUMBERVALUE OCT2BIN OCT2DEC OCT2HEX ODD ODDFPRICE ODDFYIELD ODDLPRICE ODDLYIELD OFFSET OR PDURATION PEARSON PERCENTILE.EXC PERCENTILE.INC PERCENTILE PERCENTRANK.EXC PERCENTRANK.INC PERCENTRANK PERMUT PERMUTATIONA PHI PHONETIC PI PMT POISSON.DIST POISSON POWER PPMT PRICE PRICEDISC PRICEMAT PROB PRODUCT PROPER PV QUARTILE QUARTILE.EXC QUARTILE.INC QUOTIENT RADIANS RAND RANDBETWEEN RANK.AVG RANK.EQ RANK RATE RECEIVED REGISTER.ID REPLACE REPLACEB REPT RIGHT RIGHTB ROMAN ROUND ROUNDDOWN ROUNDUP ROW ROWS RRI RSQ RTD SEARCH SEARCHB SEC SECH SECOND SERIESSUM SHEET SHEETS SIGN SIN SINH SKEW SKEW.P SLN SLOPE SMALL SQL.REQUEST SQRT SQRTPI STANDARDIZE STDEV STDEV.P STDEV.S STDEVA STDEVP STDEVPA STEYX SUBSTITUTE SUBTOTAL SUM SUMIF SUMIFS SUMPRODUCT SUMSQ SUMX2MY2 SUMX2PY2 SUMXMY2 SWITCH SYD T TAN TANH TBILLEQ TBILLPRICE TBILLYIELD T.DIST T.DIST.2T T.DIST.RT TDIST TEXT TEXTJOIN TIME TIMEVALUE T.INV T.INV.2T TINV TODAY TRANSPOSE TREND TRIM TRIMMEAN TRUE|0 TRUNC T.TEST TTEST TYPE UNICHAR UNICODE UPPER VALUE VAR VAR.P VAR.S VARA VARP VARPA VDB VLOOKUP WEBSERVICE WEEKDAY WEEKNUM WEIBULL WEIBULL.DIST WORKDAY WORKDAY.INTL XIRR XNPV XOR YEAR YEARFRAC YIELD YIELDDISC YIELDMAT Z.TEST ZTEST"},c:[{b:/^=/,e:/[^=]/,rE:!0,i:/=/,r:10},{cN:"symbol",b:/\b[A-Z]{1,2}\d+\b/,e:/[^\d]/,eE:!0,r:0},{cN:"symbol",b:/[A-Z]{0,2}\d*:[A-Z]{0,2}\d*/,r:0},e.BE,e.QSM,{cN:"number",b:e.NR+"(%)?",r:0},e.C(/\bN\(/,/\)/,{eB:!0,eE:!0,i:/\n/})]}})),e.registerLanguage("fix",(function(e){return{c:[{b:/[^\u2401\u0001]+/,e:/[\u2401\u0001]/,eE:!0,rB:!0,rE:!1,c:[{b:/([^\u2401\u0001=]+)/,e:/=([^\u2401\u0001=]+)/,rE:!0,rB:!1,cN:"attr"},{b:/=/,e:/([\u2401\u0001])/,eE:!0,eB:!0,cN:"string"}]}],cI:!0}})),e.registerLanguage("flix",(function(e){var t={cN:"string",b:/'(.|\\[xXuU][a-zA-Z0-9]+)'/},r={cN:"string",v:[{b:'"',e:'"'}]},a={cN:"title",b:/[^0-9\n\t "'(),.`{}\[\]:;][^\n\t "'(),.`{}\[\]:;]+|[^0-9\n\t "'(),.`{}\[\]:;=]/},i={cN:"function",bK:"def",e:/[:={\[(\n;]/,eE:!0,c:[a]};return{k:{literal:"true false",keyword:"case class def else enum if impl import in lat rel index let match namespace switch type yield with"},c:[e.CLCM,e.CBCM,t,r,i,e.CNM]}})),e.registerLanguage("fortran",(function(e){var t={cN:"params",b:"\\(",e:"\\)"},r={literal:".False. .True.",keyword:"kind do while private call intrinsic where elsewhere type endtype endmodule endselect endinterface end enddo endif if forall endforall only contains default return stop then public subroutine|10 function program .and. .or. .not. .le. .eq. .ge. .gt. .lt. goto save else use module select case access blank direct exist file fmt form formatted iostat name named nextrec number opened rec recl sequential status unformatted unit continue format pause cycle exit c_null_char c_alert c_backspace c_form_feed flush wait decimal round iomsg synchronous nopass non_overridable pass protected volatile abstract extends import non_intrinsic value deferred generic final enumerator class associate bind enum c_int c_short c_long c_long_long c_signed_char c_size_t c_int8_t c_int16_t c_int32_t c_int64_t c_int_least8_t c_int_least16_t c_int_least32_t c_int_least64_t c_int_fast8_t c_int_fast16_t c_int_fast32_t c_int_fast64_t c_intmax_t C_intptr_t c_float c_double c_long_double c_float_complex c_double_complex c_long_double_complex c_bool c_char c_null_ptr c_null_funptr c_new_line c_carriage_return c_horizontal_tab c_vertical_tab iso_c_binding c_loc c_funloc c_associated c_f_pointer c_ptr c_funptr iso_fortran_env character_storage_size error_unit file_storage_size input_unit iostat_end iostat_eor numeric_storage_size output_unit c_f_procpointer ieee_arithmetic ieee_support_underflow_control ieee_get_underflow_mode ieee_set_underflow_mode newunit contiguous recursive pad position action delim readwrite eor advance nml interface procedure namelist include sequence elemental pure integer real character complex logical dimension allocatable|10 parameter external implicit|10 none double precision assign intent optional pointer target in out common equivalence data",built_in:"alog alog10 amax0 amax1 amin0 amin1 amod cabs ccos cexp clog csin csqrt dabs dacos dasin datan datan2 dcos dcosh ddim dexp dint dlog dlog10 dmax1 dmin1 dmod dnint dsign dsin dsinh dsqrt dtan dtanh float iabs idim idint idnint ifix isign max0 max1 min0 min1 sngl algama cdabs cdcos cdexp cdlog cdsin cdsqrt cqabs cqcos cqexp cqlog cqsin cqsqrt dcmplx dconjg derf derfc dfloat dgamma dimag dlgama iqint qabs qacos qasin qatan qatan2 qcmplx qconjg qcos qcosh qdim qerf qerfc qexp qgamma qimag qlgama qlog qlog10 qmax1 qmin1 qmod qnint qsign qsin qsinh qsqrt qtan qtanh abs acos aimag aint anint asin atan atan2 char cmplx conjg cos cosh exp ichar index int log log10 max min nint sign sin sinh sqrt tan tanh print write dim lge lgt lle llt mod nullify allocate deallocate adjustl adjustr all allocated any associated bit_size btest ceiling count cshift date_and_time digits dot_product eoshift epsilon exponent floor fraction huge iand ibclr ibits ibset ieor ior ishft ishftc lbound len_trim matmul maxexponent maxloc maxval merge minexponent minloc minval modulo mvbits nearest pack present product radix random_number random_seed range repeat reshape rrspacing scale scan selected_int_kind selected_real_kind set_exponent shape size spacing spread sum system_clock tiny transpose trim ubound unpack verify achar iachar transfer dble entry dprod cpu_time command_argument_count get_command get_command_argument get_environment_variable is_iostat_end ieee_arithmetic ieee_support_underflow_control ieee_get_underflow_mode ieee_set_underflow_mode is_iostat_eor move_alloc new_line selected_char_kind same_type_as extends_type_ofacosh asinh atanh bessel_j0 bessel_j1 bessel_jn bessel_y0 bessel_y1 bessel_yn erf erfc erfc_scaled gamma log_gamma hypot norm2 atomic_define atomic_ref execute_command_line leadz trailz storage_size merge_bits bge bgt ble blt dshiftl dshiftr findloc iall iany iparity image_index lcobound ucobound maskl maskr num_images parity popcnt poppar shifta shiftl shiftr this_image"};return{cI:!0,aliases:["f90","f95"],k:r,i:/\/\*/,c:[e.inherit(e.ASM,{cN:"string",r:0}),e.inherit(e.QSM,{cN:"string",r:0}),{cN:"function",bK:"subroutine function program",i:"[${=\\n]",c:[e.UTM,t]},e.C("!","$",{r:0}),{cN:"number",b:"(?=\\b|\\+|\\-|\\.)(?=\\.\\d|\\d)(?:\\d+)?(?:\\.?\\d*)(?:[de][+-]?\\d+)?\\b\\.?",r:0}]}})),e.registerLanguage("fsharp",(function(e){var t={b:"<",e:">",c:[e.inherit(e.TM,{b:/'[a-zA-Z0-9_]+/})]};return{aliases:["fs"],k:"abstract and as assert base begin class default delegate do done downcast downto elif else end exception extern false finally for fun function global if in inherit inline interface internal lazy let match member module mutable namespace new null of open or override private public rec return sig static struct then to true try type upcast use val void when while with yield",i:/\/\*/,c:[{cN:"keyword",b:/\b(yield|return|let|do)!/},{cN:"string",b:'@"',e:'"',c:[{b:'""'}]},{cN:"string",b:'"""',e:'"""'},e.C("\\(\\*","\\*\\)"),{cN:"class",bK:"type",e:"\\(|=|$",eE:!0,c:[e.UTM,t]},{cN:"meta",b:"\\[<",e:">\\]",r:10},{cN:"symbol",b:"\\B('[A-Za-z])\\b",c:[e.BE]},e.CLCM,e.inherit(e.QSM,{i:null}),e.CNM]}})),e.registerLanguage("gams",(function(e){var t={keyword:"abort acronym acronyms alias all and assign binary card diag display else eq file files for free ge gt if integer le loop lt maximizing minimizing model models ne negative no not option options or ord positive prod put putpage puttl repeat sameas semicont semiint smax smin solve sos1 sos2 sum system table then until using while xor yes",literal:"eps inf na","built-in":"abs arccos arcsin arctan arctan2 Beta betaReg binomial ceil centropy cos cosh cvPower div div0 eDist entropy errorf execSeed exp fact floor frac gamma gammaReg log logBeta logGamma log10 log2 mapVal max min mod ncpCM ncpF ncpVUpow ncpVUsin normal pi poly power randBinomial randLinear randTriangle round rPower sigmoid sign signPower sin sinh slexp sllog10 slrec sqexp sqlog10 sqr sqrec sqrt tan tanh trunc uniform uniformInt vcPower bool_and bool_eqv bool_imp bool_not bool_or bool_xor ifThen rel_eq rel_ge rel_gt rel_le rel_lt rel_ne gday gdow ghour gleap gmillisec gminute gmonth gsecond gyear jdate jnow jstart jtime errorLevel execError gamsRelease gamsVersion handleCollect handleDelete handleStatus handleSubmit heapFree heapLimit heapSize jobHandle jobKill jobStatus jobTerminate licenseLevel licenseStatus maxExecError sleep timeClose timeComp timeElapsed timeExec timeStart"},r={cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0},a={cN:"symbol",v:[{b:/\=[lgenxc]=/},{b:/\$/}]},i={cN:"comment",v:[{b:"'",e:"'"},{b:'"',e:'"'}],i:"\\n",c:[e.BE]},n={b:"/",e:"/",k:t,c:[i,e.CLCM,e.CBCM,e.QSM,e.ASM,e.CNM]},o={b:/[a-z][a-z0-9_]*(\([a-z0-9_, ]*\))?[ \t]+/,eB:!0,e:"$",eW:!0,c:[i,n,{cN:"comment",b:/([ ]*[a-z0-9&#*=?@>\\<:\-,()$\[\]_.{}!+%^]+)+/,r:0}]};return{aliases:["gms"],cI:!0,k:t,c:[e.C(/^\$ontext/,/^\$offtext/),{cN:"meta",b:"^\\$[a-z0-9]+",e:"$",rB:!0,c:[{cN:"meta-keyword",b:"^\\$[a-z0-9]+"}]},e.C("^\\*","$"),e.CLCM,e.CBCM,e.QSM,e.ASM,{bK:"set sets parameter parameters variable variables scalar scalars equation equations",e:";",c:[e.C("^\\*","$"),e.CLCM,e.CBCM,e.QSM,e.ASM,n,o]},{bK:"table",e:";",rB:!0,c:[{bK:"table",e:"$",c:[o]},e.C("^\\*","$"),e.CLCM,e.CBCM,e.QSM,e.ASM,e.CNM]},{cN:"function",b:/^[a-z][a-z0-9_,\-+' ()$]+\.{2}/,rB:!0,c:[{cN:"title",b:/^[a-z0-9_]+/},r,a]},e.CNM,a]}})),e.registerLanguage("gauss",(function(e){var t={keyword:"and bool break call callexe checkinterrupt clear clearg closeall cls comlog compile continue create debug declare delete disable dlibrary dllcall do dos ed edit else elseif enable end endfor endif endp endo errorlog errorlogat expr external fn for format goto gosub graph if keyword let lib library line load loadarray loadexe loadf loadk loadm loadp loads loadx local locate loopnextindex lprint lpwidth lshow matrix msym ndpclex new not open or output outwidth plot plotsym pop prcsn print printdos proc push retp return rndcon rndmod rndmult rndseed run save saveall screen scroll setarray show sparse stop string struct system trace trap threadfor threadendfor threadbegin threadjoin threadstat threadend until use while winprint",built_in:"abs acf aconcat aeye amax amean AmericanBinomCall AmericanBinomCall_Greeks AmericanBinomCall_ImpVol AmericanBinomPut AmericanBinomPut_Greeks AmericanBinomPut_ImpVol AmericanBSCall AmericanBSCall_Greeks AmericanBSCall_ImpVol AmericanBSPut AmericanBSPut_Greeks AmericanBSPut_ImpVol amin amult annotationGetDefaults annotationSetBkd annotationSetFont annotationSetLineColor annotationSetLineStyle annotationSetLineThickness annualTradingDays arccos arcsin areshape arrayalloc arrayindex arrayinit arraytomat asciiload asclabel astd astds asum atan atan2 atranspose axmargin balance band bandchol bandcholsol bandltsol bandrv bandsolpd bar base10 begwind besselj bessely beta box boxcox cdfBeta cdfBetaInv cdfBinomial cdfBinomialInv cdfBvn cdfBvn2 cdfBvn2e cdfCauchy cdfCauchyInv cdfChic cdfChii cdfChinc cdfChincInv cdfExp cdfExpInv cdfFc cdfFnc cdfFncInv cdfGam cdfGenPareto cdfHyperGeo cdfLaplace cdfLaplaceInv cdfLogistic cdfLogisticInv cdfmControlCreate cdfMvn cdfMvn2e cdfMvnce cdfMvne cdfMvt2e cdfMvtce cdfMvte cdfN cdfN2 cdfNc cdfNegBinomial cdfNegBinomialInv cdfNi cdfPoisson cdfPoissonInv cdfRayleigh cdfRayleighInv cdfTc cdfTci cdfTnc cdfTvn cdfWeibull cdfWeibullInv cdir ceil ChangeDir chdir chiBarSquare chol choldn cholsol cholup chrs close code cols colsf combinate combinated complex con cond conj cons ConScore contour conv convertsatostr convertstrtosa corrm corrms corrvc corrx corrxs cos cosh counts countwts crossprd crout croutp csrcol csrlin csvReadM csvReadSA cumprodc cumsumc curve cvtos datacreate datacreatecomplex datalist dataload dataloop dataopen datasave date datestr datestring datestrymd dayinyr dayofweek dbAddDatabase dbClose dbCommit dbCreateQuery dbExecQuery dbGetConnectOptions dbGetDatabaseName dbGetDriverName dbGetDrivers dbGetHostName dbGetLastErrorNum dbGetLastErrorText dbGetNumericalPrecPolicy dbGetPassword dbGetPort dbGetTableHeaders dbGetTables dbGetUserName dbHasFeature dbIsDriverAvailable dbIsOpen dbIsOpenError dbOpen dbQueryBindValue dbQueryClear dbQueryCols dbQueryExecPrepared dbQueryFetchAllM dbQueryFetchAllSA dbQueryFetchOneM dbQueryFetchOneSA dbQueryFinish dbQueryGetBoundValue dbQueryGetBoundValues dbQueryGetField dbQueryGetLastErrorNum dbQueryGetLastErrorText dbQueryGetLastInsertID dbQueryGetLastQuery dbQueryGetPosition dbQueryIsActive dbQueryIsForwardOnly dbQueryIsNull dbQueryIsSelect dbQueryIsValid dbQueryPrepare dbQueryRows dbQuerySeek dbQuerySeekFirst dbQuerySeekLast dbQuerySeekNext dbQuerySeekPrevious dbQuerySetForwardOnly dbRemoveDatabase dbRollback dbSetConnectOptions dbSetDatabaseName dbSetHostName dbSetNumericalPrecPolicy dbSetPort dbSetUserName dbTransaction DeleteFile delif delrows denseToSp denseToSpRE denToZero design det detl dfft dffti diag diagrv digamma doswin DOSWinCloseall DOSWinOpen dotfeq dotfeqmt dotfge dotfgemt dotfgt dotfgtmt dotfle dotflemt dotflt dotfltmt dotfne dotfnemt draw drop dsCreate dstat dstatmt dstatmtControlCreate dtdate dtday dttime dttodtv dttostr dttoutc dtvnormal dtvtodt dtvtoutc dummy dummybr dummydn eig eigh eighv eigv elapsedTradingDays endwind envget eof eqSolve eqSolvemt eqSolvemtControlCreate eqSolvemtOutCreate eqSolveset erf erfc erfccplx erfcplx error etdays ethsec etstr EuropeanBinomCall EuropeanBinomCall_Greeks EuropeanBinomCall_ImpVol EuropeanBinomPut EuropeanBinomPut_Greeks EuropeanBinomPut_ImpVol EuropeanBSCall EuropeanBSCall_Greeks EuropeanBSCall_ImpVol EuropeanBSPut EuropeanBSPut_Greeks EuropeanBSPut_ImpVol exctsmpl exec execbg exp extern eye fcheckerr fclearerr feq feqmt fflush fft ffti fftm fftmi fftn fge fgemt fgets fgetsa fgetsat fgetst fgt fgtmt fileinfo filesa fle flemt floor flt fltmt fmod fne fnemt fonts fopen formatcv formatnv fputs fputst fseek fstrerror ftell ftocv ftos ftostrC gamma gammacplx gammaii gausset gdaAppend gdaCreate gdaDStat gdaDStatMat gdaGetIndex gdaGetName gdaGetNames gdaGetOrders gdaGetType gdaGetTypes gdaGetVarInfo gdaIsCplx gdaLoad gdaPack gdaRead gdaReadByIndex gdaReadSome gdaReadSparse gdaReadStruct gdaReportVarInfo gdaSave gdaUpdate gdaUpdateAndPack gdaVars gdaWrite gdaWrite32 gdaWriteSome getarray getdims getf getGAUSShome getmatrix getmatrix4D getname getnamef getNextTradingDay getNextWeekDay getnr getorders getpath getPreviousTradingDay getPreviousWeekDay getRow getscalar3D getscalar4D getTrRow getwind glm gradcplx gradMT gradMTm gradMTT gradMTTm gradp graphprt graphset hasimag header headermt hess hessMT hessMTg hessMTgw hessMTm hessMTmw hessMTT hessMTTg hessMTTgw hessMTTm hessMTw hessp hist histf histp hsec imag indcv indexcat indices indices2 indicesf indicesfn indnv indsav integrate1d integrateControlCreate intgrat2 intgrat3 inthp1 inthp2 inthp3 inthp4 inthpControlCreate intquad1 intquad2 intquad3 intrleav intrleavsa intrsect intsimp inv invpd invswp iscplx iscplxf isden isinfnanmiss ismiss key keyav keyw lag lag1 lagn lapEighb lapEighi lapEighvb lapEighvi lapgEig lapgEigh lapgEighv lapgEigv lapgSchur lapgSvdcst lapgSvds lapgSvdst lapSvdcusv lapSvds lapSvdusv ldlp ldlsol linSolve listwise ln lncdfbvn lncdfbvn2 lncdfmvn lncdfn lncdfn2 lncdfnc lnfact lngammacplx lnpdfmvn lnpdfmvt lnpdfn lnpdft loadd loadstruct loadwind loess loessmt loessmtControlCreate log loglog logx logy lower lowmat lowmat1 ltrisol lu lusol machEpsilon make makevars makewind margin matalloc matinit mattoarray maxbytes maxc maxindc maxv maxvec mbesselei mbesselei0 mbesselei1 mbesseli mbesseli0 mbesseli1 meanc median mergeby mergevar minc minindc minv miss missex missrv moment momentd movingave movingaveExpwgt movingaveWgt nextindex nextn nextnevn nextwind ntos null null1 numCombinations ols olsmt olsmtControlCreate olsqr olsqr2 olsqrmt ones optn optnevn orth outtyp pacf packedToSp packr parse pause pdfCauchy pdfChi pdfExp pdfGenPareto pdfHyperGeo pdfLaplace pdfLogistic pdfn pdfPoisson pdfRayleigh pdfWeibull pi pinv pinvmt plotAddArrow plotAddBar plotAddBox plotAddHist plotAddHistF plotAddHistP plotAddPolar plotAddScatter plotAddShape plotAddTextbox plotAddTS plotAddXY plotArea plotBar plotBox plotClearLayout plotContour plotCustomLayout plotGetDefaults plotHist plotHistF plotHistP plotLayout plotLogLog plotLogX plotLogY plotOpenWindow plotPolar plotSave plotScatter plotSetAxesPen plotSetBar plotSetBarFill plotSetBarStacked plotSetBkdColor plotSetFill plotSetGrid plotSetLegend plotSetLineColor plotSetLineStyle plotSetLineSymbol plotSetLineThickness plotSetNewWindow plotSetTitle plotSetWhichYAxis plotSetXAxisShow plotSetXLabel plotSetXRange plotSetXTicInterval plotSetXTicLabel plotSetYAxisShow plotSetYLabel plotSetYRange plotSetZAxisShow plotSetZLabel plotSurface plotTS plotXY polar polychar polyeval polygamma polyint polymake polymat polymroot polymult polyroot pqgwin previousindex princomp printfm printfmt prodc psi putarray putf putvals pvCreate pvGetIndex pvGetParNames pvGetParVector pvLength pvList pvPack pvPacki pvPackm pvPackmi pvPacks pvPacksi pvPacksm pvPacksmi pvPutParVector pvTest pvUnpack QNewton QNewtonmt QNewtonmtControlCreate QNewtonmtOutCreate QNewtonSet QProg QProgmt QProgmtInCreate qqr qqre qqrep qr qre qrep qrsol qrtsol qtyr qtyre qtyrep quantile quantiled qyr qyre qyrep qz rank rankindx readr real reclassify reclassifyCuts recode recserar recsercp recserrc rerun rescale reshape rets rev rfft rffti rfftip rfftn rfftnp rfftp rndBernoulli rndBeta rndBinomial rndCauchy rndChiSquare rndCon rndCreateState rndExp rndGamma rndGeo rndGumbel rndHyperGeo rndi rndKMbeta rndKMgam rndKMi rndKMn rndKMnb rndKMp rndKMu rndKMvm rndLaplace rndLCbeta rndLCgam rndLCi rndLCn rndLCnb rndLCp rndLCu rndLCvm rndLogNorm rndMTu rndMVn rndMVt rndn rndnb rndNegBinomial rndp rndPoisson rndRayleigh rndStateSkip rndu rndvm rndWeibull rndWishart rotater round rows rowsf rref sampleData satostrC saved saveStruct savewind scale scale3d scalerr scalinfnanmiss scalmiss schtoc schur searchsourcepath seekr select selif seqa seqm setdif setdifsa setvars setvwrmode setwind shell shiftr sin singleindex sinh sleep solpd sortc sortcc sortd sorthc sorthcc sortind sortindc sortmc sortr sortrc spBiconjGradSol spChol spConjGradSol spCreate spDenseSubmat spDiagRvMat spEigv spEye spLDL spline spLU spNumNZE spOnes spreadSheetReadM spreadSheetReadSA spreadSheetWrite spScale spSubmat spToDense spTrTDense spTScalar spZeros sqpSolve sqpSolveMT sqpSolveMTControlCreate sqpSolveMTlagrangeCreate sqpSolveMToutCreate sqpSolveSet sqrt statements stdc stdsc stocv stof strcombine strindx strlen strput strrindx strsect strsplit strsplitPad strtodt strtof strtofcplx strtriml strtrimr strtrunc strtruncl strtruncpad strtruncr submat subscat substute subvec sumc sumr surface svd svd1 svd2 svdcusv svds svdusv sysstate tab tan tanh tempname threadBegin threadEnd threadEndFor threadFor threadJoin threadStat time timedt timestr timeutc title tkf2eps tkf2ps tocart todaydt toeplitz token topolar trapchk trigamma trimr trunc type typecv typef union unionsa uniqindx uniqindxsa unique uniquesa upmat upmat1 upper utctodt utctodtv utrisol vals varCovMS varCovXS varget vargetl varmall varmares varput varputl vartypef vcm vcms vcx vcxs vec vech vecr vector vget view viewxyz vlist vnamecv volume vput vread vtypecv wait waitc walkindex where window writer xlabel xlsGetSheetCount xlsGetSheetSize xlsGetSheetTypes xlsMakeRange xlsReadM xlsReadSA xlsWrite xlsWriteM xlsWriteSA xpnd xtics xy xyz ylabel ytics zeros zeta zlabel ztics cdfEmpirical dot h5create h5open h5read h5readAttribute h5write h5writeAttribute ldl plotAddErrorBar plotAddSurface plotCDFEmpirical plotSetColormap plotSetContourLabels plotSetLegendFont plotSetTextInterpreter plotSetXTicCount plotSetYTicCount plotSetZLevels powerm strjoin strtrim sylvester",literal:"DB_AFTER_LAST_ROW DB_ALL_TABLES DB_BATCH_OPERATIONS DB_BEFORE_FIRST_ROW DB_BLOB DB_EVENT_NOTIFICATIONS DB_FINISH_QUERY DB_HIGH_PRECISION DB_LAST_INSERT_ID DB_LOW_PRECISION_DOUBLE DB_LOW_PRECISION_INT32 DB_LOW_PRECISION_INT64 DB_LOW_PRECISION_NUMBERS DB_MULTIPLE_RESULT_SETS DB_NAMED_PLACEHOLDERS DB_POSITIONAL_PLACEHOLDERS DB_PREPARED_QUERIES DB_QUERY_SIZE DB_SIMPLE_LOCKING DB_SYSTEM_TABLES DB_TABLES DB_TRANSACTIONS DB_UNICODE DB_VIEWS"},r={cN:"meta",b:"#",e:"$",k:{"meta-keyword":"define definecs|10 undef ifdef ifndef iflight ifdllcall ifmac ifos2win ifunix else endif lineson linesoff srcfile srcline"},c:[{b:/\\\n/,r:0},{bK:"include",e:"$",k:{"meta-keyword":"include"},c:[{cN:"meta-string",b:'"',e:'"',i:"\\n"}]},e.CLCM,e.CBCM]},a=e.UIR+"\\s*\\(?",i=[{cN:"params",b:/\(/,e:/\)/,k:t,r:0,c:[e.CNM,e.CLCM,e.CBCM]}];return{aliases:["gss"],cI:!0,k:t,i:"(\\{[%#]|[%#]\\})",c:[e.CNM,e.CLCM,e.CBCM,e.C("@","@"),r,{cN:"string",b:'"',e:'"',c:[e.BE]},{cN:"function",bK:"proc keyword",e:";",eE:!0,k:t,c:[{b:a,rB:!0,c:[e.UTM],r:0},e.CNM,e.CLCM,e.CBCM,r].concat(i)},{cN:"function",bK:"fn",e:";",eE:!0,k:t,c:[{b:a+e.IR+"\\)?\\s*\\=\\s*",rB:!0,c:[e.UTM],r:0},e.CNM,e.CLCM,e.CBCM].concat(i)},{cN:"function",b:"\\bexternal (proc|keyword|fn)\\s+",e:";",eE:!0,k:t,c:[{b:a,rB:!0,c:[e.UTM],r:0},e.CLCM,e.CBCM]},{cN:"function",b:"\\bexternal (matrix|string|array|sparse matrix|struct "+e.IR+")\\s+",e:";",eE:!0,k:t,c:[e.CLCM,e.CBCM]}]}})),e.registerLanguage("gcode",(function(e){var t="[A-Z_][A-Z0-9_.]*",r="\\%",a="IF DO WHILE ENDWHILE CALL ENDIF SUB ENDSUB GOTO REPEAT ENDREPEAT EQ LT GT NE GE LE OR XOR",i={cN:"meta",b:"([O])([0-9]+)"},n=[e.CLCM,e.CBCM,e.C(/\(/,/\)/),e.inherit(e.CNM,{b:"([-+]?([0-9]*\\.?[0-9]+\\.?))|"+e.CNR}),e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null}),{cN:"name",b:"([G])([0-9]+\\.?[0-9]?)"},{cN:"name",b:"([M])([0-9]+\\.?[0-9]?)"},{cN:"attr",b:"(VC|VS|#)",e:"(\\d+)"},{cN:"attr",b:"(VZOFX|VZOFY|VZOFZ)"},{cN:"built_in",b:"(ATAN|ABS|ACOS|ASIN|SIN|COS|EXP|FIX|FUP|ROUND|LN|TAN)(\\[)",e:"([-+]?([0-9]*\\.?[0-9]+\\.?))(\\])"},{cN:"symbol",v:[{b:"N",e:"\\d+",i:"\\W"}]}];return{aliases:["nc"],cI:!0,l:t,k:a,c:[{cN:"meta",b:r},i].concat(n)}})),e.registerLanguage("gherkin",(function(e){return{aliases:["feature"],k:"Feature Background Ability Business Need Scenario Scenarios Scenario Outline Scenario Template Examples Given And Then But When",c:[{cN:"symbol",b:"\\*",r:0},{cN:"meta",b:"@[^@\\s]+"},{b:"\\|",e:"\\|\\w*$",c:[{cN:"string",b:"[^|]+"}]},{cN:"variable",b:"<",e:">"},e.HCM,{cN:"string",b:'"""',e:'"""'},e.QSM]}})),e.registerLanguage("glsl",(function(e){return{k:{keyword:"break continue discard do else for if return while switch case default attribute binding buffer ccw centroid centroid varying coherent column_major const cw depth_any depth_greater depth_less depth_unchanged early_fragment_tests equal_spacing flat fractional_even_spacing fractional_odd_spacing highp in index inout invariant invocations isolines layout line_strip lines lines_adjacency local_size_x local_size_y local_size_z location lowp max_vertices mediump noperspective offset origin_upper_left out packed patch pixel_center_integer point_mode points precise precision quads r11f_g11f_b10f r16 r16_snorm r16f r16i r16ui r32f r32i r32ui r8 r8_snorm r8i r8ui readonly restrict rg16 rg16_snorm rg16f rg16i rg16ui rg32f rg32i rg32ui rg8 rg8_snorm rg8i rg8ui rgb10_a2 rgb10_a2ui rgba16 rgba16_snorm rgba16f rgba16i rgba16ui rgba32f rgba32i rgba32ui rgba8 rgba8_snorm rgba8i rgba8ui row_major sample shared smooth std140 std430 stream triangle_strip triangles triangles_adjacency uniform varying vertices volatile writeonly",type:"atomic_uint bool bvec2 bvec3 bvec4 dmat2 dmat2x2 dmat2x3 dmat2x4 dmat3 dmat3x2 dmat3x3 dmat3x4 dmat4 dmat4x2 dmat4x3 dmat4x4 double dvec2 dvec3 dvec4 float iimage1D iimage1DArray iimage2D iimage2DArray iimage2DMS iimage2DMSArray iimage2DRect iimage3D iimageBufferiimageCube iimageCubeArray image1D image1DArray image2D image2DArray image2DMS image2DMSArray image2DRect image3D imageBuffer imageCube imageCubeArray int isampler1D isampler1DArray isampler2D isampler2DArray isampler2DMS isampler2DMSArray isampler2DRect isampler3D isamplerBuffer isamplerCube isamplerCubeArray ivec2 ivec3 ivec4 mat2 mat2x2 mat2x3 mat2x4 mat3 mat3x2 mat3x3 mat3x4 mat4 mat4x2 mat4x3 mat4x4 sampler1D sampler1DArray sampler1DArrayShadow sampler1DShadow sampler2D sampler2DArray sampler2DArrayShadow sampler2DMS sampler2DMSArray sampler2DRect sampler2DRectShadow sampler2DShadow sampler3D samplerBuffer samplerCube samplerCubeArray samplerCubeArrayShadow samplerCubeShadow image1D uimage1DArray uimage2D uimage2DArray uimage2DMS uimage2DMSArray uimage2DRect uimage3D uimageBuffer uimageCube uimageCubeArray uint usampler1D usampler1DArray usampler2D usampler2DArray usampler2DMS usampler2DMSArray usampler2DRect usampler3D samplerBuffer usamplerCube usamplerCubeArray uvec2 uvec3 uvec4 vec2 vec3 vec4 void",built_in:"gl_MaxAtomicCounterBindings gl_MaxAtomicCounterBufferSize gl_MaxClipDistances gl_MaxClipPlanes gl_MaxCombinedAtomicCounterBuffers gl_MaxCombinedAtomicCounters gl_MaxCombinedImageUniforms gl_MaxCombinedImageUnitsAndFragmentOutputs gl_MaxCombinedTextureImageUnits gl_MaxComputeAtomicCounterBuffers gl_MaxComputeAtomicCounters gl_MaxComputeImageUniforms gl_MaxComputeTextureImageUnits gl_MaxComputeUniformComponents gl_MaxComputeWorkGroupCount gl_MaxComputeWorkGroupSize gl_MaxDrawBuffers gl_MaxFragmentAtomicCounterBuffers gl_MaxFragmentAtomicCounters gl_MaxFragmentImageUniforms gl_MaxFragmentInputComponents gl_MaxFragmentInputVectors gl_MaxFragmentUniformComponents gl_MaxFragmentUniformVectors gl_MaxGeometryAtomicCounterBuffers gl_MaxGeometryAtomicCounters gl_MaxGeometryImageUniforms gl_MaxGeometryInputComponents gl_MaxGeometryOutputComponents gl_MaxGeometryOutputVertices gl_MaxGeometryTextureImageUnits gl_MaxGeometryTotalOutputComponents gl_MaxGeometryUniformComponents gl_MaxGeometryVaryingComponents gl_MaxImageSamples gl_MaxImageUnits gl_MaxLights gl_MaxPatchVertices gl_MaxProgramTexelOffset gl_MaxTessControlAtomicCounterBuffers gl_MaxTessControlAtomicCounters gl_MaxTessControlImageUniforms gl_MaxTessControlInputComponents gl_MaxTessControlOutputComponents gl_MaxTessControlTextureImageUnits gl_MaxTessControlTotalOutputComponents gl_MaxTessControlUniformComponents gl_MaxTessEvaluationAtomicCounterBuffers gl_MaxTessEvaluationAtomicCounters gl_MaxTessEvaluationImageUniforms gl_MaxTessEvaluationInputComponents gl_MaxTessEvaluationOutputComponents gl_MaxTessEvaluationTextureImageUnits gl_MaxTessEvaluationUniformComponents gl_MaxTessGenLevel gl_MaxTessPatchComponents gl_MaxTextureCoords gl_MaxTextureImageUnits gl_MaxTextureUnits gl_MaxVaryingComponents gl_MaxVaryingFloats gl_MaxVaryingVectors gl_MaxVertexAtomicCounterBuffers gl_MaxVertexAtomicCounters gl_MaxVertexAttribs gl_MaxVertexImageUniforms gl_MaxVertexOutputComponents gl_MaxVertexOutputVectors gl_MaxVertexTextureImageUnits gl_MaxVertexUniformComponents gl_MaxVertexUniformVectors gl_MaxViewports gl_MinProgramTexelOffset gl_BackColor gl_BackLightModelProduct gl_BackLightProduct gl_BackMaterial gl_BackSecondaryColor gl_ClipDistance gl_ClipPlane gl_ClipVertex gl_Color gl_DepthRange gl_EyePlaneQ gl_EyePlaneR gl_EyePlaneS gl_EyePlaneT gl_Fog gl_FogCoord gl_FogFragCoord gl_FragColor gl_FragCoord gl_FragData gl_FragDepth gl_FrontColor gl_FrontFacing gl_FrontLightModelProduct gl_FrontLightProduct gl_FrontMaterial gl_FrontSecondaryColor gl_GlobalInvocationID gl_InstanceID gl_InvocationID gl_Layer gl_LightModel gl_LightSource gl_LocalInvocationID gl_LocalInvocationIndex gl_ModelViewMatrix gl_ModelViewMatrixInverse gl_ModelViewMatrixInverseTranspose gl_ModelViewMatrixTranspose gl_ModelViewProjectionMatrix gl_ModelViewProjectionMatrixInverse gl_ModelViewProjectionMatrixInverseTranspose gl_ModelViewProjectionMatrixTranspose gl_MultiTexCoord0 gl_MultiTexCoord1 gl_MultiTexCoord2 gl_MultiTexCoord3 gl_MultiTexCoord4 gl_MultiTexCoord5 gl_MultiTexCoord6 gl_MultiTexCoord7 gl_Normal gl_NormalMatrix gl_NormalScale gl_NumSamples gl_NumWorkGroups gl_ObjectPlaneQ gl_ObjectPlaneR gl_ObjectPlaneS gl_ObjectPlaneT gl_PatchVerticesIn gl_Point gl_PointCoord gl_PointSize gl_Position gl_PrimitiveID gl_PrimitiveIDIn gl_ProjectionMatrix gl_ProjectionMatrixInverse gl_ProjectionMatrixInverseTranspose gl_ProjectionMatrixTranspose gl_SampleID gl_SampleMask gl_SampleMaskIn gl_SamplePosition gl_SecondaryColor gl_TessCoord gl_TessLevelInner gl_TessLevelOuter gl_TexCoord gl_TextureEnvColor gl_TextureMatrix gl_TextureMatrixInverse gl_TextureMatrixInverseTranspose gl_TextureMatrixTranspose gl_Vertex gl_VertexID gl_ViewportIndex gl_WorkGroupID gl_WorkGroupSize gl_in gl_out EmitStreamVertex EmitVertex EndPrimitive EndStreamPrimitive abs acos acosh all any asin asinh atan atanh atomicAdd atomicAnd atomicCompSwap atomicCounter atomicCounterDecrement atomicCounterIncrement atomicExchange atomicMax atomicMin atomicOr atomicXor barrier bitCount bitfieldExtract bitfieldInsert bitfieldReverse ceil clamp cos cosh cross dFdx dFdy degrees determinant distance dot equal exp exp2 faceforward findLSB findMSB floatBitsToInt floatBitsToUint floor fma fract frexp ftransform fwidth greaterThan greaterThanEqual groupMemoryBarrier imageAtomicAdd imageAtomicAnd imageAtomicCompSwap imageAtomicExchange imageAtomicMax imageAtomicMin imageAtomicOr imageAtomicXor imageLoad imageSize imageStore imulExtended intBitsToFloat interpolateAtCentroid interpolateAtOffset interpolateAtSample inverse inversesqrt isinf isnan ldexp length lessThan lessThanEqual log log2 matrixCompMult max memoryBarrier memoryBarrierAtomicCounter memoryBarrierBuffer memoryBarrierImage memoryBarrierShared min mix mod modf noise1 noise2 noise3 noise4 normalize not notEqual outerProduct packDouble2x32 packHalf2x16 packSnorm2x16 packSnorm4x8 packUnorm2x16 packUnorm4x8 pow radians reflect refract round roundEven shadow1D shadow1DLod shadow1DProj shadow1DProjLod shadow2D shadow2DLod shadow2DProj shadow2DProjLod sign sin sinh smoothstep sqrt step tan tanh texelFetch texelFetchOffset texture texture1D texture1DLod texture1DProj texture1DProjLod texture2D texture2DLod texture2DProj texture2DProjLod texture3D texture3DLod texture3DProj texture3DProjLod textureCube textureCubeLod textureGather textureGatherOffset textureGatherOffsets textureGrad textureGradOffset textureLod textureLodOffset textureOffset textureProj textureProjGrad textureProjGradOffset textureProjLod textureProjLodOffset textureProjOffset textureQueryLevels textureQueryLod textureSize transpose trunc uaddCarry uintBitsToFloat umulExtended unpackDouble2x32 unpackHalf2x16 unpackSnorm2x16 unpackSnorm4x8 unpackUnorm2x16 unpackUnorm4x8 usubBorrow",literal:"true false"},i:'"',c:[e.CLCM,e.CBCM,e.CNM,{cN:"meta",b:"#",e:"$"}]}})),e.registerLanguage("go",(function(e){var t={keyword:"break default func interface select case map struct chan else goto package switch const fallthrough if range type continue for import return var go defer bool byte complex64 complex128 float32 float64 int8 int16 int32 int64 string uint8 uint16 uint32 uint64 int uint uintptr rune",literal:"true false iota nil",built_in:"append cap close complex copy imag len make new panic print println real recover delete"};return{aliases:["golang"],k:t,i:"",e:",\\s+",rB:!0,eW:!0,c:[{cN:"attr",b:":\\w+"},e.ASM,e.QSM,{b:"\\w+",r:0}]}]},{b:"\\(\\s*",e:"\\s*\\)",eE:!0,c:[{b:"\\w+\\s*=",e:"\\s+",rB:!0,eW:!0,c:[{cN:"attr",b:"\\w+",r:0},e.ASM,e.QSM,{b:"\\w+",r:0}]}]}]},{b:"^\\s*[=~]\\s*"},{b:"#{",starts:{e:"}",sL:"ruby"}}]}})),e.registerLanguage("handlebars",(function(e){var t={"builtin-name":"each in with if else unless bindattr action collection debugger log outlet template unbound view yield"};return{aliases:["hbs","html.hbs","html.handlebars"],cI:!0,sL:"xml",c:[e.C("{{!(--)?","(--)?}}"),{cN:"template-tag",b:/\{\{[#\/]/,e:/\}\}/,c:[{cN:"name",b:/[a-zA-Z\.-]+/,k:t,starts:{eW:!0,r:0,c:[e.QSM]}}]},{cN:"template-variable",b:/\{\{/,e:/\}\}/,k:t}]}})),e.registerLanguage("haskell",(function(e){var t={v:[e.C("--","$"),e.C("{-","-}",{c:["self"]})]},r={cN:"meta",b:"{-#",e:"#-}"},a={cN:"meta",b:"^#",e:"$"},i={cN:"type",b:"\\b[A-Z][\\w']*",r:0},n={b:"\\(",e:"\\)",i:'"',c:[r,a,{cN:"type",b:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},e.inherit(e.TM,{b:"[_a-z][\\w']*"}),t]},o={b:"{",e:"}",c:n.c};return{aliases:["hs"],k:"let in if then else case of where do module import hiding qualified type data newtype deriving class instance as default infix infixl infixr foreign export ccall stdcall cplusplus jvm dotnet safe unsafe family forall mdo proc rec",c:[{bK:"module",e:"where",k:"module where",c:[n,t],i:"\\W\\.|;"},{b:"\\bimport\\b",e:"$",k:"import qualified as hiding",c:[n,t],i:"\\W\\.|;"},{cN:"class",b:"^(\\s*)?(class|instance)\\b",e:"where",k:"class family instance where",c:[i,n,t]},{cN:"class",b:"\\b(data|(new)?type)\\b",e:"$",k:"data family type newtype deriving",c:[r,i,n,o,t]},{bK:"default",e:"$",c:[i,n,t]},{bK:"infix infixl infixr",e:"$",c:[e.CNM,t]},{b:"\\bforeign\\b",e:"$",k:"foreign import export ccall stdcall cplusplus jvm dotnet safe unsafe",c:[i,e.QSM,t]},{cN:"meta",b:"#!\\/usr\\/bin\\/env runhaskell",e:"$"},r,a,e.QSM,e.CNM,i,e.inherit(e.TM,{b:"^[_a-z][\\w']*"}),t,{b:"->|<-"}]}})),e.registerLanguage("haxe",(function(e){var t="Int Float String Bool Dynamic Void Array ";return{aliases:["hx"],k:{keyword:"break case cast catch continue default do dynamic else enum extern for function here if import in inline never new override package private get set public return static super switch this throw trace try typedef untyped using var while "+t,built_in:"trace this",literal:"true false null _"},c:[{cN:"string",b:"'",e:"'",c:[e.BE,{cN:"subst",b:"\\$\\{",e:"\\}"},{cN:"subst",b:"\\$",e:"\\W}"}]},e.QSM,e.CLCM,e.CBCM,e.CNM,{cN:"meta",b:"@:",e:"$"},{cN:"meta",b:"#",e:"$",k:{"meta-keyword":"if else elseif end error"}},{cN:"type",b:":[ \t]*",e:"[^A-Za-z0-9_ \t\\->]",eB:!0,eE:!0,r:0},{cN:"type",b:":[ \t]*",e:"\\W",eB:!0,eE:!0},{cN:"type",b:"new *",e:"\\W",eB:!0,eE:!0},{cN:"class",bK:"enum",e:"\\{",c:[e.TM]},{cN:"class",bK:"abstract",e:"[\\{$]",c:[{cN:"type",b:"\\(",e:"\\)",eB:!0,eE:!0},{cN:"type",b:"from +",e:"\\W",eB:!0,eE:!0},{cN:"type",b:"to +",e:"\\W",eB:!0,eE:!0},e.TM],k:{keyword:"abstract from to"}},{cN:"class",b:"\\b(class|interface) +",e:"[\\{$]",eE:!0,k:"class interface",c:[{cN:"keyword",b:"\\b(extends|implements) +",k:"extends implements",c:[{cN:"type",b:e.IR,r:0}]},e.TM]},{cN:"function",bK:"function",e:"\\(",eE:!0,i:"\\S",c:[e.TM]}],i:/<\//}})),e.registerLanguage("hsp",(function(e){return{cI:!0,l:/[\w\._]+/,k:"goto gosub return break repeat loop continue wait await dim sdim foreach dimtype dup dupptr end stop newmod delmod mref run exgoto on mcall assert logmes newlab resume yield onexit onerror onkey onclick oncmd exist delete mkdir chdir dirlist bload bsave bcopy memfile if else poke wpoke lpoke getstr chdpm memexpand memcpy memset notesel noteadd notedel noteload notesave randomize noteunsel noteget split strrep setease button chgdisp exec dialog mmload mmplay mmstop mci pset pget syscolor mes print title pos circle cls font sysfont objsize picload color palcolor palette redraw width gsel gcopy gzoom gmode bmpsave hsvcolor getkey listbox chkbox combox input mesbox buffer screen bgscr mouse objsel groll line clrobj boxf objprm objmode stick grect grotate gsquare gradf objimage objskip objenable celload celdiv celput newcom querycom delcom cnvstow comres axobj winobj sendmsg comevent comevarg sarrayconv callfunc cnvwtos comevdisp libptr system hspstat hspver stat cnt err strsize looplev sublev iparam wparam lparam refstr refdval int rnd strlen length length2 length3 length4 vartype gettime peek wpeek lpeek varptr varuse noteinfo instr abs limit getease str strmid strf getpath strtrim sin cos tan atan sqrt double absf expf logf limitf powf geteasef mousex mousey mousew hwnd hinstance hdc ginfo objinfo dirinfo sysinfo thismod __hspver__ __hsp30__ __date__ __time__ __line__ __file__ _debug __hspdef__ and or xor not screen_normal screen_palette screen_hide screen_fixedsize screen_tool screen_frame gmode_gdi gmode_mem gmode_rgb0 gmode_alpha gmode_rgb0alpha gmode_add gmode_sub gmode_pixela ginfo_mx ginfo_my ginfo_act ginfo_sel ginfo_wx1 ginfo_wy1 ginfo_wx2 ginfo_wy2 ginfo_vx ginfo_vy ginfo_sizex ginfo_sizey ginfo_winx ginfo_winy ginfo_mesx ginfo_mesy ginfo_r ginfo_g ginfo_b ginfo_paluse ginfo_dispx ginfo_dispy ginfo_cx ginfo_cy ginfo_intid ginfo_newid ginfo_sx ginfo_sy objinfo_mode objinfo_bmscr objinfo_hwnd notemax notesize dir_cur dir_exe dir_win dir_sys dir_cmdline dir_desktop dir_mydoc dir_tv font_normal font_bold font_italic font_underline font_strikeout font_antialias objmode_normal objmode_guifont objmode_usefont gsquare_grad msgothic msmincho do until while wend for next _break _continue switch case default swbreak swend ddim ldim alloc m_pi rad2deg deg2rad ease_linear ease_quad_in ease_quad_out ease_quad_inout ease_cubic_in ease_cubic_out ease_cubic_inout ease_quartic_in ease_quartic_out ease_quartic_inout ease_bounce_in ease_bounce_out ease_bounce_inout ease_shake_in ease_shake_out ease_shake_inout ease_loop",c:[e.CLCM,e.CBCM,e.QSM,e.ASM,{cN:"string",b:'{"',e:'"}',c:[e.BE]},e.C(";","$",{r:0}),{cN:"meta",b:"#",e:"$",k:{"meta-keyword":"addion cfunc cmd cmpopt comfunc const defcfunc deffunc define else endif enum epack func global if ifdef ifndef include modcfunc modfunc modinit modterm module pack packopt regcmd runtime undef usecom uselib"},c:[e.inherit(e.QSM,{cN:"meta-string"}),e.NM,e.CNM,e.CLCM,e.CBCM]},{cN:"symbol",b:"^\\*(\\w+|@)"},e.NM,e.CNM]}})),e.registerLanguage("htmlbars",(function(e){var t="action collection component concat debugger each each-in else get hash if input link-to loc log mut outlet partial query-params render textarea unbound unless with yield view",r={i:/\}\}/,b:/[a-zA-Z0-9_]+=/,rB:!0,r:0,c:[{cN:"attr",b:/[a-zA-Z0-9_]+/}]},a=({i:/\}\}/,b:/\)/,e:/\)/,c:[{b:/[a-zA-Z\.\-]+/,k:{built_in:t},starts:{eW:!0,r:0,c:[e.QSM]}}]},{eW:!0,r:0,k:{keyword:"as",built_in:t},c:[e.QSM,r,e.NM]});return{cI:!0,sL:"xml",c:[e.C("{{!(--)?","(--)?}}"),{cN:"template-tag",b:/\{\{[#\/]/,e:/\}\}/,c:[{cN:"name",b:/[a-zA-Z\.\-]+/,k:{"builtin-name":t},starts:a}]},{cN:"template-variable",b:/\{\{[a-zA-Z][a-zA-Z\-]+/,e:/\}\}/,k:{keyword:"as",built_in:t},c:[e.QSM]}]}})),e.registerLanguage("http",(function(e){var t="HTTP/[0-9\\.]+";return{aliases:["https"],i:"\\S",c:[{b:"^"+t,e:"$",c:[{cN:"number",b:"\\b\\d{3}\\b"}]},{b:"^[A-Z]+ (.*?) "+t+"$",rB:!0,e:"$",c:[{cN:"string",b:" ",e:" ",eB:!0,eE:!0},{b:t},{cN:"keyword",b:"[A-Z]+"}]},{cN:"attribute",b:"^\\w",e:": ",eE:!0,i:"\\n|\\s|=",starts:{e:"$",r:0}},{b:"\\n\\n",starts:{sL:[],eW:!0}}]}})),e.registerLanguage("hy",(function(e){var t={"builtin-name":"!= % %= & &= * ** **= *= *map + += , --build-class-- --import-- -= . / // //= /= < << <<= <= = > >= >> >>= @ @= ^ ^= abs accumulate all and any ap-compose ap-dotimes ap-each ap-each-while ap-filter ap-first ap-if ap-last ap-map ap-map-when ap-pipe ap-reduce ap-reject apply as-> ascii assert assoc bin break butlast callable calling-module-name car case cdr chain chr coll? combinations compile compress cond cons cons? continue count curry cut cycle dec def default-method defclass defmacro defmacro-alias defmacro/g! defmain defmethod defmulti defn defn-alias defnc defnr defreader defseq del delattr delete-route dict-comp dir disassemble dispatch-reader-macro distinct divmod do doto drop drop-last drop-while empty? end-sequence eval eval-and-compile eval-when-compile even? every? except exec filter first flatten float? fn fnc fnr for for* format fraction genexpr gensym get getattr global globals group-by hasattr hash hex id identity if if* if-not if-python2 import in inc input instance? integer integer-char? integer? interleave interpose is is-coll is-cons is-empty is-even is-every is-float is-instance is-integer is-integer-char is-iterable is-iterator is-keyword is-neg is-none is-not is-numeric is-odd is-pos is-string is-symbol is-zero isinstance islice issubclass iter iterable? iterate iterator? keyword keyword? lambda last len let lif lif-not list* list-comp locals loop macro-error macroexpand macroexpand-1 macroexpand-all map max merge-with method-decorator min multi-decorator multicombinations name neg? next none? nonlocal not not-in not? nth numeric? oct odd? open or ord partition permutations pos? post-route postwalk pow prewalk print product profile/calls profile/cpu put-route quasiquote quote raise range read read-str recursive-replace reduce remove repeat repeatedly repr require rest round route route-with-methods rwm second seq set-comp setattr setv some sorted string string? sum switch symbol? take take-nth take-while tee try unless unquote unquote-splicing vars walk when while with with* with-decorator with-gensyms xi xor yield yield-from zero? zip zip-longest | |= ~"},r="a-zA-Z_\\-!.?+*=<>&#'",a="["+r+"]["+r+"0-9/;:]*",i="[-+]?\\d+(\\.\\d+)?",n={cN:"meta",b:"^#!",e:"$"},o={b:a,r:0},s={cN:"number",b:i,r:0},l=e.inherit(e.QSM,{i:null}),c=e.C(";","$",{r:0}),d={cN:"literal",b:/\b([Tt]rue|[Ff]else|nil|None)\b/},p={b:"[\\[\\{]",e:"[\\]\\}]"},m={cN:"comment",b:"\\^"+a},u=e.C("\\^\\{","\\}"),b={cN:"symbol",b:"[:]{1,2}"+a},g={b:"\\(",e:"\\)"},f={eW:!0,r:0},_={k:t,l:a,cN:"name",b:a,starts:f},h=[g,l,m,u,c,b,p,s,d,o];return g.c=[e.C("comment",""),_,f],f.c=h,p.c=h,{aliases:["hylang"],i:/\S/,c:[n,g,l,m,u,c,b,p,s,d]}})),e.registerLanguage("inform7",(function(e){var t="\\[",r="\\]";return{aliases:["i7"],cI:!0,k:{keyword:"thing room person man woman animal container supporter backdrop door scenery open closed locked inside gender is are say understand kind of rule"},c:[{cN:"string",b:'"',e:'"',r:0,c:[{cN:"subst",b:t,e:r}]},{cN:"section",b:/^(Volume|Book|Part|Chapter|Section|Table)\b/,e:"$"},{b:/^(Check|Carry out|Report|Instead of|To|Rule|When|Before|After)\b/,e:":",c:[{b:"\\(This",e:"\\)"}]},{cN:"comment",b:t,e:r,c:["self"]}]}})),e.registerLanguage("ini",(function(e){var t={cN:"string",c:[e.BE],v:[{b:"'''",e:"'''",r:10},{b:'"""',e:'"""',r:10},{b:'"',e:'"'},{b:"'",e:"'"}]};return{aliases:["toml"],cI:!0,i:/\S/,c:[e.C(";","$"),e.HCM,{cN:"section",b:/^\s*\[+/,e:/\]+/},{b:/^[a-z0-9\[\]_-]+\s*=\s*/,e:"$",rB:!0,c:[{cN:"attr",b:/[a-z0-9\[\]_-]+/},{b:/=/,eW:!0,r:0,c:[{cN:"literal",b:/\bon|off|true|false|yes|no\b/},{cN:"variable",v:[{b:/\$[\w\d"][\w\d_]*/},{b:/\$\{(.*?)}/}]},t,{cN:"number",b:/([\+\-]+)?[\d]+_[\d_]+/},e.NM]}]}]}})),e.registerLanguage("irpf90",(function(e){var t={cN:"params",b:"\\(",e:"\\)"},r={literal:".False. .True.",keyword:"kind do while private call intrinsic where elsewhere type endtype endmodule endselect endinterface end enddo endif if forall endforall only contains default return stop then public subroutine|10 function program .and. .or. .not. .le. .eq. .ge. .gt. .lt. goto save else use module select case access blank direct exist file fmt form formatted iostat name named nextrec number opened rec recl sequential status unformatted unit continue format pause cycle exit c_null_char c_alert c_backspace c_form_feed flush wait decimal round iomsg synchronous nopass non_overridable pass protected volatile abstract extends import non_intrinsic value deferred generic final enumerator class associate bind enum c_int c_short c_long c_long_long c_signed_char c_size_t c_int8_t c_int16_t c_int32_t c_int64_t c_int_least8_t c_int_least16_t c_int_least32_t c_int_least64_t c_int_fast8_t c_int_fast16_t c_int_fast32_t c_int_fast64_t c_intmax_t C_intptr_t c_float c_double c_long_double c_float_complex c_double_complex c_long_double_complex c_bool c_char c_null_ptr c_null_funptr c_new_line c_carriage_return c_horizontal_tab c_vertical_tab iso_c_binding c_loc c_funloc c_associated c_f_pointer c_ptr c_funptr iso_fortran_env character_storage_size error_unit file_storage_size input_unit iostat_end iostat_eor numeric_storage_size output_unit c_f_procpointer ieee_arithmetic ieee_support_underflow_control ieee_get_underflow_mode ieee_set_underflow_mode newunit contiguous recursive pad position action delim readwrite eor advance nml interface procedure namelist include sequence elemental pure integer real character complex logical dimension allocatable|10 parameter external implicit|10 none double precision assign intent optional pointer target in out common equivalence data begin_provider &begin_provider end_provider begin_shell end_shell begin_template end_template subst assert touch soft_touch provide no_dep free irp_if irp_else irp_endif irp_write irp_read",built_in:"alog alog10 amax0 amax1 amin0 amin1 amod cabs ccos cexp clog csin csqrt dabs dacos dasin datan datan2 dcos dcosh ddim dexp dint dlog dlog10 dmax1 dmin1 dmod dnint dsign dsin dsinh dsqrt dtan dtanh float iabs idim idint idnint ifix isign max0 max1 min0 min1 sngl algama cdabs cdcos cdexp cdlog cdsin cdsqrt cqabs cqcos cqexp cqlog cqsin cqsqrt dcmplx dconjg derf derfc dfloat dgamma dimag dlgama iqint qabs qacos qasin qatan qatan2 qcmplx qconjg qcos qcosh qdim qerf qerfc qexp qgamma qimag qlgama qlog qlog10 qmax1 qmin1 qmod qnint qsign qsin qsinh qsqrt qtan qtanh abs acos aimag aint anint asin atan atan2 char cmplx conjg cos cosh exp ichar index int log log10 max min nint sign sin sinh sqrt tan tanh print write dim lge lgt lle llt mod nullify allocate deallocate adjustl adjustr all allocated any associated bit_size btest ceiling count cshift date_and_time digits dot_product eoshift epsilon exponent floor fraction huge iand ibclr ibits ibset ieor ior ishft ishftc lbound len_trim matmul maxexponent maxloc maxval merge minexponent minloc minval modulo mvbits nearest pack present product radix random_number random_seed range repeat reshape rrspacing scale scan selected_int_kind selected_real_kind set_exponent shape size spacing spread sum system_clock tiny transpose trim ubound unpack verify achar iachar transfer dble entry dprod cpu_time command_argument_count get_command get_command_argument get_environment_variable is_iostat_end ieee_arithmetic ieee_support_underflow_control ieee_get_underflow_mode ieee_set_underflow_mode is_iostat_eor move_alloc new_line selected_char_kind same_type_as extends_type_ofacosh asinh atanh bessel_j0 bessel_j1 bessel_jn bessel_y0 bessel_y1 bessel_yn erf erfc erfc_scaled gamma log_gamma hypot norm2 atomic_define atomic_ref execute_command_line leadz trailz storage_size merge_bits bge bgt ble blt dshiftl dshiftr findloc iall iany iparity image_index lcobound ucobound maskl maskr num_images parity popcnt poppar shifta shiftl shiftr this_image IRP_ALIGN irp_here"};return{cI:!0,k:r,i:/\/\*/,c:[e.inherit(e.ASM,{cN:"string",r:0}),e.inherit(e.QSM,{cN:"string",r:0}),{cN:"function",bK:"subroutine function program",i:"[${=\\n]",c:[e.UTM,t]},e.C("!","$",{r:0}),e.C("begin_doc","end_doc",{r:10}),{cN:"number",b:"(?=\\b|\\+|\\-|\\.)(?=\\.\\d|\\d)(?:\\d+)?(?:\\.?\\d*)(?:[de][+-]?\\d+)?\\b\\.?",r:0}]}})),e.registerLanguage("java",(function(e){var t="[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",r=t+"(<"+t+"(\\s*,\\s*"+t+")*>)?",a="false synchronized int abstract float private char boolean static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",i="\\b(0[bB]([01]+[01_]+[01]+|[01]+)|0[xX]([a-fA-F0-9]+[a-fA-F0-9_]+[a-fA-F0-9]+|[a-fA-F0-9]+)|(([\\d]+[\\d_]+[\\d]+|[\\d]+)(\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))?|\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))([eE][-+]?\\d+)?)[lLfF]?",n={cN:"number",b:i,r:0};return{aliases:["jsp"],k:a,i:/<\/|#/,c:[e.C("/\\*\\*","\\*/",{r:0,c:[{b:/\w+@/,r:0},{cN:"doctag",b:"@[A-Za-z]+"}]}),e.CLCM,e.CBCM,e.ASM,e.QSM,{cN:"class",bK:"class interface",e:/[{;=]/,eE:!0,k:"class interface",i:/[:"\[\]]/,c:[{bK:"extends implements"},e.UTM]},{bK:"new throw return else",r:0},{cN:"function",b:"("+r+"\\s+)+"+e.UIR+"\\s*\\(",rB:!0,e:/[{;=]/,eE:!0,k:a,c:[{b:e.UIR+"\\s*\\(",rB:!0,r:0,c:[e.UTM]},{cN:"params",b:/\(/,e:/\)/,k:a,r:0,c:[e.ASM,e.QSM,e.CNM,e.CBCM]},e.CLCM,e.CBCM]},n,{cN:"meta",b:"@[A-Za-z]+"}]}})),e.registerLanguage("javascript",(function(e){var t="[A-Za-z$_][0-9A-Za-z$_]*",r={keyword:"in of if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await static import from as",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Promise"},a={cN:"number",v:[{b:"\\b(0[bB][01]+)"},{b:"\\b(0[oO][0-7]+)"},{b:e.CNR}],r:0},i={cN:"subst",b:"\\$\\{",e:"\\}",k:r,c:[]},n={cN:"string",b:"`",e:"`",c:[e.BE,i]};i.c=[e.ASM,e.QSM,n,a,e.RM];var o=i.c.concat([e.CBCM,e.CLCM]);return{aliases:["js","jsx"],k:r,c:[{cN:"meta",r:10,b:/^\s*['"]use (strict|asm)['"]/},{cN:"meta",b:/^#!/,e:/$/},e.ASM,e.QSM,n,e.CLCM,e.CBCM,a,{b:/[{,]\s*/,r:0,c:[{b:t+"\\s*:",rB:!0,r:0,c:[{cN:"attr",b:t,r:0}]}]},{b:"("+e.RSR+"|\\b(case|return|throw)\\b)\\s*",k:"return throw case",c:[e.CLCM,e.CBCM,e.RM,{cN:"function",b:"(\\(.*?\\)|"+t+")\\s*=>",rB:!0,e:"\\s*=>",c:[{cN:"params",v:[{b:t},{b:/\(\s*\)/},{b:/\(/,e:/\)/,eB:!0,eE:!0,k:r,c:o}]}]},{b://,sL:"xml",c:[{b:/<\w+\s*\/>/,skip:!0},{b:/<\w+/,e:/(\/\w+|\w+\/)>/,skip:!0,c:[{b:/<\w+\s*\/>/,skip:!0},"self"]}]}],r:0},{cN:"function",bK:"function",e:/\{/,eE:!0,c:[e.inherit(e.TM,{b:t}),{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,c:o}],i:/\[|%/},{b:/\$[(.]/},e.METHOD_GUARD,{cN:"class",bK:"class",e:/[{;=]/,eE:!0,i:/[:"\[\]]/,c:[{bK:"extends"},e.UTM]},{bK:"constructor",e:/\{/,eE:!0}],i:/#(?!!)/}})),e.registerLanguage("jboss-cli",(function(e){var t={b:/[\w-]+ *=/,rB:!0,r:0,c:[{cN:"attr",b:/[\w-]+/}]},r={cN:"params",b:/\(/,e:/\)/,c:[t],r:0},a={cN:"function",b:/:[\w\-.]+/,r:0},i={cN:"string",b:/\B(([\/.])[\w\-.\/=]+)+/},n={cN:"params",b:/--[\w\-=\/]+/};return{aliases:["wildfly-cli"],l:"[a-z-]+",k:{keyword:"alias batch cd clear command connect connection-factory connection-info data-source deploy deployment-info deployment-overlay echo echo-dmr help history if jdbc-driver-info jms-queue|20 jms-topic|20 ls patch pwd quit read-attribute read-operation reload rollout-plan run-batch set shutdown try unalias undeploy unset version xa-data-source",literal:"true false"},c:[e.HCM,e.QSM,n,a,i,r]}})),e.registerLanguage("json",(function(e){var t={literal:"true false null"},r=[e.QSM,e.CNM],a={e:",",eW:!0,eE:!0,c:r,k:t},i={b:"{",e:"}",c:[{cN:"attr",b:/"/,e:/"/,c:[e.BE],i:"\\n"},e.inherit(a,{b:/:/})],i:"\\S"},n={b:"\\[",e:"\\]",c:[e.inherit(a)],i:"\\S"};return r.splice(r.length,0,i,n),{c:r,k:t,i:"\\S"}})),e.registerLanguage("julia",(function(e){var t={keyword:"in isa where baremodule begin break catch ccall const continue do else elseif end export false finally for function global if import importall let local macro module quote return true try using while type immutable abstract bitstype typealias ",literal:"true false ARGS C_NULL DevNull ENDIAN_BOM ENV I Inf Inf16 Inf32 Inf64 InsertionSort JULIA_HOME LOAD_PATH MergeSort NaN NaN16 NaN32 NaN64 PROGRAM_FILE QuickSort RoundDown RoundFromZero RoundNearest RoundNearestTiesAway RoundNearestTiesUp RoundToZero RoundUp STDERR STDIN STDOUT VERSION catalan e|0 eu|0 eulergamma golden im nothing pi γ Ï€ φ ",built_in:"ANY AbstractArray AbstractChannel AbstractFloat AbstractMatrix AbstractRNG AbstractSerializer AbstractSet AbstractSparseArray AbstractSparseMatrix AbstractSparseVector AbstractString AbstractUnitRange AbstractVecOrMat AbstractVector Any ArgumentError Array AssertionError Associative Base64DecodePipe Base64EncodePipe Bidiagonal BigFloat BigInt BitArray BitMatrix BitVector Bool BoundsError BufferStream CachingPool CapturedException CartesianIndex CartesianRange Cchar Cdouble Cfloat Channel Char Cint Cintmax_t Clong Clonglong ClusterManager Cmd CodeInfo Colon Complex Complex128 Complex32 Complex64 CompositeException Condition ConjArray ConjMatrix ConjVector Cptrdiff_t Cshort Csize_t Cssize_t Cstring Cuchar Cuint Cuintmax_t Culong Culonglong Cushort Cwchar_t Cwstring DataType Date DateFormat DateTime DenseArray DenseMatrix DenseVecOrMat DenseVector Diagonal Dict DimensionMismatch Dims DirectIndexString Display DivideError DomainError EOFError EachLine Enum Enumerate ErrorException Exception ExponentialBackOff Expr Factorization FileMonitor Float16 Float32 Float64 Function Future GlobalRef GotoNode HTML Hermitian IO IOBuffer IOContext IOStream IPAddr IPv4 IPv6 IndexCartesian IndexLinear IndexStyle InexactError InitError Int Int128 Int16 Int32 Int64 Int8 IntSet Integer InterruptException InvalidStateException Irrational KeyError LabelNode LinSpace LineNumberNode LoadError LowerTriangular MIME Matrix MersenneTwister Method MethodError MethodTable Module NTuple NewvarNode NullException Nullable Number ObjectIdDict OrdinalRange OutOfMemoryError OverflowError Pair ParseError PartialQuickSort PermutedDimsArray Pipe PollingFileWatcher ProcessExitedException Ptr QuoteNode RandomDevice Range RangeIndex Rational RawFD ReadOnlyMemoryError Real ReentrantLock Ref Regex RegexMatch RemoteChannel RemoteException RevString RoundingMode RowVector SSAValue SegmentationFault SerializationState Set SharedArray SharedMatrix SharedVector Signed SimpleVector Slot SlotNumber SparseMatrixCSC SparseVector StackFrame StackOverflowError StackTrace StepRange StepRangeLen StridedArray StridedMatrix StridedVecOrMat StridedVector String SubArray SubString SymTridiagonal Symbol Symmetric SystemError TCPSocket Task Text TextDisplay Timer Tridiagonal Tuple Type TypeError TypeMapEntry TypeMapLevel TypeName TypeVar TypedSlot UDPSocket UInt UInt128 UInt16 UInt32 UInt64 UInt8 UndefRefError UndefVarError UnicodeError UniformScaling Union UnionAll UnitRange Unsigned UpperTriangular Val Vararg VecElement VecOrMat Vector VersionNumber Void WeakKeyDict WeakRef WorkerConfig WorkerPool "},r="[A-Za-z_\\u00A1-\\uFFFF][A-Za-z_0-9\\u00A1-\\uFFFF]*",a={l:r,k:t,i:/<\//},i={cN:"number",b:/(\b0x[\d_]*(\.[\d_]*)?|0x\.\d[\d_]*)p[-+]?\d+|\b0[box][a-fA-F0-9][a-fA-F0-9_]*|(\b\d[\d_]*(\.[\d_]*)?|\.\d[\d_]*)([eEfF][-+]?\d+)?/,r:0},n={cN:"string",b:/'(.|\\[xXuU][a-zA-Z0-9]+)'/},o={cN:"subst",b:/\$\(/,e:/\)/,k:t},s={cN:"variable",b:"\\$"+r},l={cN:"string",c:[e.BE,o,s],v:[{b:/\w*"""/,e:/"""\w*/,r:10},{b:/\w*"/,e:/"\w*/}]},c={cN:"string",c:[e.BE,o,s],b:"`",e:"`"},d={cN:"meta",b:"@"+r},p={cN:"comment",v:[{b:"#=",e:"=#",r:10},{b:"#",e:"$"}]};return a.c=[i,n,l,c,d,p,e.HCM,{cN:"keyword",b:"\\b(((abstract|primitive)\\s+)type|(mutable\\s+)?struct)\\b"},{b:/<:/}],o.c=a.c,a})),e.registerLanguage("julia-repl",(function(e){return{c:[{cN:"meta",b:/^julia>/,r:10,starts:{e:/^(?![ ]{6})/,sL:"julia"},aliases:["jldoctest"]}]}})),e.registerLanguage("kotlin",(function(e){var t={keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit initinterface annotation data sealed internal infix operator out by constructor super trait volatile transient native default",built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",literal:"true false null"},r={cN:"keyword",b:/\b(break|continue|return|this)\b/,starts:{c:[{cN:"symbol",b:/@\w+/}]}},a={cN:"symbol",b:e.UIR+"@"},i={cN:"subst",b:"\\${",e:"}",c:[e.ASM,e.CNM]},n={cN:"variable",b:"\\$"+e.UIR},o={cN:"string",v:[{b:'"""',e:'"""',c:[n,i]},{b:"'",e:"'",i:/\n/,c:[e.BE]},{b:'"',e:'"',i:/\n/,c:[e.BE,n,i]}]},s={cN:"meta",b:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UIR+")?"},l={cN:"meta",b:"@"+e.UIR,c:[{b:/\(/,e:/\)/,c:[e.inherit(o,{cN:"meta-string"})]}]};return{k:t,c:[e.C("/\\*\\*","\\*/",{r:0,c:[{cN:"doctag",b:"@[A-Za-z]+"}]}),e.CLCM,e.CBCM,r,a,s,l,{cN:"function",bK:"fun",e:"[(]|$",rB:!0,eE:!0,k:t,i:/fun\s+(<.*>)?[^\s\(]+(\s+[^\s\(]+)\s*=/,r:5,c:[{b:e.UIR+"\\s*\\(",rB:!0,r:0,c:[e.UTM]},{cN:"type",b://,k:"reified",r:0},{cN:"params",b:/\(/,e:/\)/,endsParent:!0,k:t,r:0,c:[{b:/:/,e:/[=,\/]/,eW:!0,c:[{cN:"type",b:e.UIR},e.CLCM,e.CBCM],r:0},e.CLCM,e.CBCM,s,l,o,e.CNM]},e.CBCM]},{cN:"class",bK:"class interface trait",e:/[:\{(]|$/,eE:!0,i:"extends implements",c:[{bK:"public protected internal private constructor"},e.UTM,{cN:"type",b://,eB:!0,eE:!0,r:0},{cN:"type",b:/[,:]\s*/,e:/[<\(,]|$/,eB:!0,rE:!0},s,l]},o,{cN:"meta",b:"^#!/usr/bin/env",e:"$",i:"\n"},e.CNM]}})),e.registerLanguage("lasso",(function(e){var t="[a-zA-Z_][\\w.]*",r="<\\?(lasso(script)?|=)",a="\\]|\\?>",i={literal:"true false none minimal full all void and or not bw nbw ew new cn ncn lt lte gt gte eq neq rx nrx ft",built_in:"array date decimal duration integer map pair string tag xml null boolean bytes keyword list locale queue set stack staticarray local var variable global data self inherited currentcapture givenblock",keyword:"cache database_names database_schemanames database_tablenames define_tag define_type email_batch encode_set html_comment handle handle_error header if inline iterate ljax_target link link_currentaction link_currentgroup link_currentrecord link_detail link_firstgroup link_firstrecord link_lastgroup link_lastrecord link_nextgroup link_nextrecord link_prevgroup link_prevrecord log loop namespace_using output_none portal private protect records referer referrer repeating resultset rows search_args search_arguments select sort_args sort_arguments thread_atomic value_list while abort case else fail_if fail_ifnot fail if_empty if_false if_null if_true loop_abort loop_continue loop_count params params_up return return_value run_children soap_definetag soap_lastrequest soap_lastresponse tag_name ascending average by define descending do equals frozen group handle_failure import in into join let match max min on order parent protected provide public require returnhome skip split_thread sum take thread to trait type where with yield yieldhome"},n=e.C("\x3c!--","--\x3e",{r:0}),o={cN:"meta",b:"\\[noprocess\\]",starts:{e:"\\[/noprocess\\]",rE:!0,c:[n]}},s={cN:"meta",b:"\\[/noprocess|"+r},l={cN:"symbol",b:"'"+t+"'"},c=[e.CLCM,e.CBCM,e.inherit(e.CNM,{b:e.CNR+"|(-?infinity|NaN)\\b"}),e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null}),{cN:"string",b:"`",e:"`"},{v:[{b:"[#$]"+t},{b:"#",e:"\\d+",i:"\\W"}]},{cN:"type",b:"::\\s*",e:t,i:"\\W"},{cN:"params",v:[{b:"-(?!infinity)"+t,r:0},{b:"(\\.\\.\\.)"}]},{b:/(->|\.)\s*/,r:0,c:[l]},{cN:"class",bK:"define",rE:!0,e:"\\(|=>",c:[e.inherit(e.TM,{b:t+"(=(?!>))?|[-+*/%](?!>)"})]}];return{aliases:["ls","lassoscript"],cI:!0,l:t+"|&[lg]t;",k:i,c:[{cN:"meta",b:a,r:0,starts:{e:"\\[|"+r,rE:!0,r:0,c:[n]}},o,s,{cN:"meta",b:"\\[no_square_brackets",starts:{e:"\\[/no_square_brackets\\]",l:t+"|&[lg]t;",k:i,c:[{cN:"meta",b:a,r:0,starts:{e:"\\[noprocess\\]|"+r,rE:!0,c:[n]}},o,s].concat(c)}},{cN:"meta",b:"\\[",r:0},{cN:"meta",b:"^#!",e:"lasso9$",r:10}].concat(c)}})),e.registerLanguage("ldif",(function(e){return{c:[{cN:"attribute",b:"^dn",e:": ",eE:!0,starts:{e:"$",r:0},r:10},{cN:"attribute",b:"^\\w",e:": ",eE:!0,starts:{e:"$",r:0}},{cN:"literal",b:"^-",e:"$"},e.HCM]}})),e.registerLanguage("leaf",(function(e){return{c:[{cN:"function",b:"#+[A-Za-z_0-9]*\\(",e:" {",rB:!0,eE:!0,c:[{cN:"keyword",b:"#+"},{cN:"title",b:"[A-Za-z_][A-Za-z_0-9]*"},{cN:"params",b:"\\(",e:"\\)",endsParent:!0,c:[{cN:"string",b:'"',e:'"'},{cN:"variable",b:"[A-Za-z_][A-Za-z_0-9]*"}]}]}]}})),e.registerLanguage("less",(function(e){var t="[\\w-]+",r="("+t+"|@{"+t+"})",a=[],i=[],n=function(e){return{cN:"string",b:"~?"+e+".*?"+e}},o=function(e,t,r){return{cN:e,b:t,r:r}},s={b:"\\(",e:"\\)",c:i,r:0};i.push(e.CLCM,e.CBCM,n("'"),n('"'),e.CSSNM,{b:"(url|data-uri)\\(",starts:{cN:"string",e:"[\\)\\n]",eE:!0}},o("number","#[0-9A-Fa-f]+\\b"),s,o("variable","@@?"+t,10),o("variable","@{"+t+"}"),o("built_in","~?`[^`]*?`"),{cN:"attribute",b:t+"\\s*:",e:":",rB:!0,eE:!0},{cN:"meta",b:"!important"});var l=i.concat({b:"{",e:"}",c:a}),c={bK:"when",eW:!0,c:[{bK:"and not"}].concat(i)},d={b:r+"\\s*:",rB:!0,e:"[;}]",r:0,c:[{cN:"attribute",b:r,e:":",eE:!0,starts:{eW:!0,i:"[<=$]",r:0,c:i}}]},p={cN:"keyword",b:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b",starts:{e:"[;{}]",rE:!0,c:i,r:0}},m={cN:"variable",v:[{b:"@"+t+"\\s*:",r:15},{b:"@"+t}],starts:{e:"[;}]",rE:!0,c:l}},u={v:[{b:"[\\.#:&\\[>]",e:"[;{}]"},{b:r,e:"{"}],rB:!0,rE:!0,i:"[<='$\"]",r:0,c:[e.CLCM,e.CBCM,c,o("keyword","all\\b"),o("variable","@{"+t+"}"),o("selector-tag",r+"%?",0),o("selector-id","#"+r),o("selector-class","\\."+r,0),o("selector-tag","&",0),{cN:"selector-attr",b:"\\[",e:"\\]"},{cN:"selector-pseudo",b:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{b:"\\(",e:"\\)",c:l},{b:"!important"}]};return a.push(e.CLCM,e.CBCM,p,m,d,u),{cI:!0,i:"[=>'/<($\"]",c:a}})),e.registerLanguage("lisp",(function(e){var t="[a-zA-Z_\\-\\+\\*\\/\\<\\=\\>\\&\\#][a-zA-Z0-9_\\-\\+\\*\\/\\<\\=\\>\\&\\#!]*",r="\\|[^]*?\\|",a="(\\-|\\+)?\\d+(\\.\\d+|\\/\\d+)?((d|e|f|l|s|D|E|F|L|S)(\\+|\\-)?\\d+)?",i={cN:"meta",b:"^#!",e:"$"},n={cN:"literal",b:"\\b(t{1}|nil)\\b"},o={cN:"number",v:[{b:a,r:0},{b:"#(b|B)[0-1]+(/[0-1]+)?"},{b:"#(o|O)[0-7]+(/[0-7]+)?"},{b:"#(x|X)[0-9a-fA-F]+(/[0-9a-fA-F]+)?"},{b:"#(c|C)\\("+a+" +"+a,e:"\\)"}]},s=e.inherit(e.QSM,{i:null}),l=e.C(";","$",{r:0}),c={b:"\\*",e:"\\*"},d={cN:"symbol",b:"[:&]"+t},p={b:t,r:0},m={b:r},u={b:"\\(",e:"\\)",c:["self",n,s,o,p]},b={c:[o,s,c,d,u,p],v:[{b:"['`]\\(",e:"\\)"},{b:"\\(quote ",e:"\\)",k:{name:"quote"}},{b:"'"+r}]},g={v:[{b:"'"+t},{b:"#'"+t+"(::"+t+")*"}]},f={b:"\\(\\s*",e:"\\)"},_={eW:!0,r:0};return f.c=[{cN:"name",v:[{b:t},{b:r}]},_],_.c=[b,g,f,n,o,s,l,c,d,m,p],{i:/\S/,c:[o,i,n,s,l,b,g,f,p]}})),e.registerLanguage("livecodeserver",(function(e){var t={b:"\\b[gtps][A-Z]+[A-Za-z0-9_\\-]*\\b|\\$_[A-Z]+",r:0},r=[e.CBCM,e.HCM,e.C("--","$"),e.C("[^:]//","$")],a=e.inherit(e.TM,{v:[{b:"\\b_*rig[A-Z]+[A-Za-z0-9_\\-]*"},{b:"\\b_[a-z0-9\\-]+"}]}),i=e.inherit(e.TM,{b:"\\b([A-Za-z0-9_\\-]+)\\b"});return{cI:!1,k:{keyword:"$_COOKIE $_FILES $_GET $_GET_BINARY $_GET_RAW $_POST $_POST_BINARY $_POST_RAW $_SESSION $_SERVER codepoint codepoints segment segments codeunit codeunits sentence sentences trueWord trueWords paragraph after byte bytes english the until http forever descending using line real8 with seventh for stdout finally element word words fourth before black ninth sixth characters chars stderr uInt1 uInt1s uInt2 uInt2s stdin string lines relative rel any fifth items from middle mid at else of catch then third it file milliseconds seconds second secs sec int1 int1s int4 int4s internet int2 int2s normal text item last long detailed effective uInt4 uInt4s repeat end repeat URL in try into switch to words https token binfile each tenth as ticks tick system real4 by dateItems without char character ascending eighth whole dateTime numeric short first ftp integer abbreviated abbr abbrev private case while if div mod wrap and or bitAnd bitNot bitOr bitXor among not in a an within contains ends with begins the keys of keys",literal:"SIX TEN FORMFEED NINE ZERO NONE SPACE FOUR FALSE COLON CRLF PI COMMA ENDOFFILE EOF EIGHT FIVE QUOTE EMPTY ONE TRUE RETURN CR LINEFEED RIGHT BACKSLASH NULL SEVEN TAB THREE TWO six ten formfeed nine zero none space four false colon crlf pi comma endoffile eof eight five quote empty one true return cr linefeed right backslash null seven tab three two RIVERSION RISTATE FILE_READ_MODE FILE_WRITE_MODE FILE_WRITE_MODE DIR_WRITE_MODE FILE_READ_UMASK FILE_WRITE_UMASK DIR_READ_UMASK DIR_WRITE_UMASK",built_in:"put abs acos aliasReference annuity arrayDecode arrayEncode asin atan atan2 average avg avgDev base64Decode base64Encode baseConvert binaryDecode binaryEncode byteOffset byteToNum cachedURL cachedURLs charToNum cipherNames codepointOffset codepointProperty codepointToNum codeunitOffset commandNames compound compress constantNames cos date dateFormat decompress directories diskSpace DNSServers exp exp1 exp2 exp10 extents files flushEvents folders format functionNames geometricMean global globals hasMemory harmonicMean hostAddress hostAddressToName hostName hostNameToAddress isNumber ISOToMac itemOffset keys len length libURLErrorData libUrlFormData libURLftpCommand libURLLastHTTPHeaders libURLLastRHHeaders libUrlMultipartFormAddPart libUrlMultipartFormData libURLVersion lineOffset ln ln1 localNames log log2 log10 longFilePath lower macToISO matchChunk matchText matrixMultiply max md5Digest median merge millisec millisecs millisecond milliseconds min monthNames nativeCharToNum normalizeText num number numToByte numToChar numToCodepoint numToNativeChar offset open openfiles openProcesses openProcessIDs openSockets paragraphOffset paramCount param params peerAddress pendingMessages platform popStdDev populationStandardDeviation populationVariance popVariance processID random randomBytes replaceText result revCreateXMLTree revCreateXMLTreeFromFile revCurrentRecord revCurrentRecordIsFirst revCurrentRecordIsLast revDatabaseColumnCount revDatabaseColumnIsNull revDatabaseColumnLengths revDatabaseColumnNames revDatabaseColumnNamed revDatabaseColumnNumbered revDatabaseColumnTypes revDatabaseConnectResult revDatabaseCursors revDatabaseID revDatabaseTableNames revDatabaseType revDataFromQuery revdb_closeCursor revdb_columnbynumber revdb_columncount revdb_columnisnull revdb_columnlengths revdb_columnnames revdb_columntypes revdb_commit revdb_connect revdb_connections revdb_connectionerr revdb_currentrecord revdb_cursorconnection revdb_cursorerr revdb_cursors revdb_dbtype revdb_disconnect revdb_execute revdb_iseof revdb_isbof revdb_movefirst revdb_movelast revdb_movenext revdb_moveprev revdb_query revdb_querylist revdb_recordcount revdb_rollback revdb_tablenames revGetDatabaseDriverPath revNumberOfRecords revOpenDatabase revOpenDatabases revQueryDatabase revQueryDatabaseBlob revQueryResult revQueryIsAtStart revQueryIsAtEnd revUnixFromMacPath revXMLAttribute revXMLAttributes revXMLAttributeValues revXMLChildContents revXMLChildNames revXMLCreateTreeFromFileWithNamespaces revXMLCreateTreeWithNamespaces revXMLDataFromXPathQuery revXMLEvaluateXPath revXMLFirstChild revXMLMatchingNode revXMLNextSibling revXMLNodeContents revXMLNumberOfChildren revXMLParent revXMLPreviousSibling revXMLRootNode revXMLRPC_CreateRequest revXMLRPC_Documents revXMLRPC_Error revXMLRPC_GetHost revXMLRPC_GetMethod revXMLRPC_GetParam revXMLText revXMLRPC_Execute revXMLRPC_GetParamCount revXMLRPC_GetParamNode revXMLRPC_GetParamType revXMLRPC_GetPath revXMLRPC_GetPort revXMLRPC_GetProtocol revXMLRPC_GetRequest revXMLRPC_GetResponse revXMLRPC_GetSocket revXMLTree revXMLTrees revXMLValidateDTD revZipDescribeItem revZipEnumerateItems revZipOpenArchives round sampVariance sec secs seconds sentenceOffset sha1Digest shell shortFilePath sin specialFolderPath sqrt standardDeviation statRound stdDev sum sysError systemVersion tan tempName textDecode textEncode tick ticks time to tokenOffset toLower toUpper transpose truewordOffset trunc uniDecode uniEncode upper URLDecode URLEncode URLStatus uuid value variableNames variance version waitDepth weekdayNames wordOffset xsltApplyStylesheet xsltApplyStylesheetFromFile xsltLoadStylesheet xsltLoadStylesheetFromFile add breakpoint cancel clear local variable file word line folder directory URL close socket process combine constant convert create new alias folder directory decrypt delete variable word line folder directory URL dispatch divide do encrypt filter get include intersect kill libURLDownloadToFile libURLFollowHttpRedirects libURLftpUpload libURLftpUploadFile libURLresetAll libUrlSetAuthCallback libURLSetCustomHTTPHeaders libUrlSetExpect100 libURLSetFTPListCommand libURLSetFTPMode libURLSetFTPStopTime libURLSetStatusCallback load multiply socket prepare process post seek rel relative read from process rename replace require resetAll resolve revAddXMLNode revAppendXML revCloseCursor revCloseDatabase revCommitDatabase revCopyFile revCopyFolder revCopyXMLNode revDeleteFolder revDeleteXMLNode revDeleteAllXMLTrees revDeleteXMLTree revExecuteSQL revGoURL revInsertXMLNode revMoveFolder revMoveToFirstRecord revMoveToLastRecord revMoveToNextRecord revMoveToPreviousRecord revMoveToRecord revMoveXMLNode revPutIntoXMLNode revRollBackDatabase revSetDatabaseDriverPath revSetXMLAttribute revXMLRPC_AddParam revXMLRPC_DeleteAllDocuments revXMLAddDTD revXMLRPC_Free revXMLRPC_FreeAll revXMLRPC_DeleteDocument revXMLRPC_DeleteParam revXMLRPC_SetHost revXMLRPC_SetMethod revXMLRPC_SetPort revXMLRPC_SetProtocol revXMLRPC_SetSocket revZipAddItemWithData revZipAddItemWithFile revZipAddUncompressedItemWithData revZipAddUncompressedItemWithFile revZipCancel revZipCloseArchive revZipDeleteItem revZipExtractItemToFile revZipExtractItemToVariable revZipSetProgressCallback revZipRenameItem revZipReplaceItemWithData revZipReplaceItemWithFile revZipOpenArchive send set sort split start stop subtract union unload wait write"},c:[t,{cN:"keyword",b:"\\bend\\sif\\b"},{cN:"function",bK:"function",e:"$",c:[t,i,e.ASM,e.QSM,e.BNM,e.CNM,a]},{cN:"function",b:"\\bend\\s+",e:"$",k:"end",c:[i,a],r:0},{bK:"command on",e:"$",c:[t,i,e.ASM,e.QSM,e.BNM,e.CNM,a]},{cN:"meta",v:[{b:"<\\?(rev|lc|livecode)",r:10},{b:"<\\?"},{b:"\\?>"}]},e.ASM,e.QSM,e.BNM,e.CNM,a].concat(r),i:";$|^\\[|^=|&|{"}})),e.registerLanguage("livescript",(function(e){var t={keyword:"in if for while finally new do return else break catch instanceof throw try this switch continue typeof delete debugger case default function var with then unless until loop of by when and or is isnt not it that otherwise from to til fallthrough super case default function var void const let enum export import native __hasProp __extends __slice __bind __indexOf",literal:"true false null undefined yes no on off it that void",built_in:"npm require console print module global window document"},r="[A-Za-z$_](?:-[0-9A-Za-z$_]|[0-9A-Za-z$_])*",a=e.inherit(e.TM,{b:r}),i={cN:"subst",b:/#\{/,e:/}/,k:t},n={cN:"subst",b:/#[A-Za-z$_]/,e:/(?:\-[0-9A-Za-z$_]|[0-9A-Za-z$_])*/,k:t},o=[e.BNM,{cN:"number",b:"(\\b0[xX][a-fA-F0-9_]+)|(\\b\\d(\\d|_\\d)*(\\.(\\d(\\d|_\\d)*)?)?(_*[eE]([-+]\\d(_\\d|\\d)*)?)?[_a-z]*)",r:0,starts:{e:"(\\s*/)?",r:0}},{cN:"string",v:[{b:/'''/,e:/'''/,c:[e.BE]},{b:/'/,e:/'/,c:[e.BE]},{b:/"""/,e:/"""/,c:[e.BE,i,n]},{b:/"/,e:/"/,c:[e.BE,i,n]},{b:/\\/,e:/(\s|$)/,eE:!0}]},{cN:"regexp",v:[{b:"//",e:"//[gim]*",c:[i,e.HCM]},{b:/\/(?![ *])(\\\/|.)*?\/[gim]*(?=\W|$)/}]},{b:"@"+r},{b:"``",e:"``",eB:!0,eE:!0,sL:"javascript"}];i.c=o;var s={cN:"params",b:"\\(",rB:!0,c:[{b:/\(/,e:/\)/,k:t,c:["self"].concat(o)}]};return{aliases:["ls"],k:t,i:/\/\*/,c:o.concat([e.C("\\/\\*","\\*\\/"),e.HCM,{cN:"function",c:[a,s],rB:!0,v:[{b:"("+r+"\\s*(?:=|:=)\\s*)?(\\(.*\\))?\\s*\\B\\->\\*?",e:"\\->\\*?"},{b:"("+r+"\\s*(?:=|:=)\\s*)?!?(\\(.*\\))?\\s*\\B[-~]{1,2}>\\*?",e:"[-~]{1,2}>\\*?"},{b:"("+r+"\\s*(?:=|:=)\\s*)?(\\(.*\\))?\\s*\\B!?[-~]{1,2}>\\*?",e:"!?[-~]{1,2}>\\*?"}]},{cN:"class",bK:"class",e:"$",i:/[:="\[\]]/,c:[{bK:"extends",eW:!0,i:/[:="\[\]]/,c:[a]},a]},{b:r+":",e:":",rB:!0,rE:!0,r:0}])}})),e.registerLanguage("llvm",(function(e){var t="([-a-zA-Z$._][\\w\\-$.]*)";return{k:"begin end true false declare define global constant private linker_private internal available_externally linkonce linkonce_odr weak weak_odr appending dllimport dllexport common default hidden protected extern_weak external thread_local zeroinitializer undef null to tail target triple datalayout volatile nuw nsw nnan ninf nsz arcp fast exact inbounds align addrspace section alias module asm sideeffect gc dbg linker_private_weak attributes blockaddress initialexec localdynamic localexec prefix unnamed_addr ccc fastcc coldcc x86_stdcallcc x86_fastcallcc arm_apcscc arm_aapcscc arm_aapcs_vfpcc ptx_device ptx_kernel intel_ocl_bicc msp430_intrcc spir_func spir_kernel x86_64_sysvcc x86_64_win64cc x86_thiscallcc cc c signext zeroext inreg sret nounwind noreturn noalias nocapture byval nest readnone readonly inlinehint noinline alwaysinline optsize ssp sspreq noredzone noimplicitfloat naked builtin cold nobuiltin noduplicate nonlazybind optnone returns_twice sanitize_address sanitize_memory sanitize_thread sspstrong uwtable returned type opaque eq ne slt sgt sle sge ult ugt ule uge oeq one olt ogt ole oge ord uno ueq une x acq_rel acquire alignstack atomic catch cleanup filter inteldialect max min monotonic nand personality release seq_cst singlethread umax umin unordered xchg add fadd sub fsub mul fmul udiv sdiv fdiv urem srem frem shl lshr ashr and or xor icmp fcmp phi call trunc zext sext fptrunc fpext uitofp sitofp fptoui fptosi inttoptr ptrtoint bitcast addrspacecast select va_arg ret br switch invoke unwind unreachable indirectbr landingpad resume malloc alloca free load store getelementptr extractelement insertelement shufflevector getresult extractvalue insertvalue atomicrmw cmpxchg fence argmemonly double",c:[{cN:"keyword",b:"i\\d+"},e.C(";","\\n",{r:0}),e.QSM,{cN:"string",v:[{b:'"',e:'[^\\\\]"'}],r:0},{cN:"title",v:[{b:"@"+t},{b:"@\\d+"},{b:"!"+t},{b:"!\\d+"+t}]},{cN:"symbol",v:[{b:"%"+t},{b:"%\\d+"},{b:"#\\d+"}]},{cN:"number",v:[{b:"0[xX][a-fA-F0-9]+"},{b:"-?\\d+(?:[.]\\d+)?(?:[eE][-+]?\\d+(?:[.]\\d+)?)?"}],r:0}]}})),e.registerLanguage("lsl",(function(e){var t={cN:"subst",b:/\\[tn"\\]/},r={cN:"string",b:'"',e:'"',c:[t]},a={cN:"number",b:e.CNR},i={cN:"literal",v:[{b:"\\b(?:PI|TWO_PI|PI_BY_TWO|DEG_TO_RAD|RAD_TO_DEG|SQRT2)\\b"},{b:"\\b(?:XP_ERROR_(?:EXPERIENCES_DISABLED|EXPERIENCE_(?:DISABLED|SUSPENDED)|INVALID_(?:EXPERIENCE|PARAMETERS)|KEY_NOT_FOUND|MATURITY_EXCEEDED|NONE|NOT_(?:FOUND|PERMITTED(?:_LAND)?)|NO_EXPERIENCE|QUOTA_EXCEEDED|RETRY_UPDATE|STORAGE_EXCEPTION|STORE_DISABLED|THROTTLED|UNKNOWN_ERROR)|JSON_APPEND|STATUS_(?:PHYSICS|ROTATE_[XYZ]|PHANTOM|SANDBOX|BLOCK_GRAB(?:_OBJECT)?|(?:DIE|RETURN)_AT_EDGE|CAST_SHADOWS|OK|MALFORMED_PARAMS|TYPE_MISMATCH|BOUNDS_ERROR|NOT_(?:FOUND|SUPPORTED)|INTERNAL_ERROR|WHITELIST_FAILED)|AGENT(?:_(?:BY_(?:LEGACY_|USER)NAME|FLYING|ATTACHMENTS|SCRIPTED|MOUSELOOK|SITTING|ON_OBJECT|AWAY|WALKING|IN_AIR|TYPING|CROUCHING|BUSY|ALWAYS_RUN|AUTOPILOT|LIST_(?:PARCEL(?:_OWNER)?|REGION)))?|CAMERA_(?:PITCH|DISTANCE|BEHINDNESS_(?:ANGLE|LAG)|(?:FOCUS|POSITION)(?:_(?:THRESHOLD|LOCKED|LAG))?|FOCUS_OFFSET|ACTIVE)|ANIM_ON|LOOP|REVERSE|PING_PONG|SMOOTH|ROTATE|SCALE|ALL_SIDES|LINK_(?:ROOT|SET|ALL_(?:OTHERS|CHILDREN)|THIS)|ACTIVE|PASS(?:IVE|_(?:ALWAYS|IF_NOT_HANDLED|NEVER))|SCRIPTED|CONTROL_(?:FWD|BACK|(?:ROT_)?(?:LEFT|RIGHT)|UP|DOWN|(?:ML_)?LBUTTON)|PERMISSION_(?:RETURN_OBJECTS|DEBIT|OVERRIDE_ANIMATIONS|SILENT_ESTATE_MANAGEMENT|TAKE_CONTROLS|TRIGGER_ANIMATION|ATTACH|CHANGE_LINKS|(?:CONTROL|TRACK)_CAMERA|TELEPORT)|INVENTORY_(?:TEXTURE|SOUND|OBJECT|SCRIPT|LANDMARK|CLOTHING|NOTECARD|BODYPART|ANIMATION|GESTURE|ALL|NONE)|CHANGED_(?:INVENTORY|COLOR|SHAPE|SCALE|TEXTURE|LINK|ALLOWED_DROP|OWNER|REGION(?:_START)?|TELEPORT|MEDIA)|OBJECT_(?:CLICK_ACTION|HOVER_HEIGHT|LAST_OWNER_ID|(?:PHYSICS|SERVER|STREAMING)_COST|UNKNOWN_DETAIL|CHARACTER_TIME|PHANTOM|PHYSICS|TEMP_ON_REZ|NAME|DESC|POS|PRIM_(?:COUNT|EQUIVALENCE)|RETURN_(?:PARCEL(?:_OWNER)?|REGION)|REZZER_KEY|ROO?T|VELOCITY|OMEGA|OWNER|GROUP|CREATOR|ATTACHED_POINT|RENDER_WEIGHT|(?:BODY_SHAPE|PATHFINDING)_TYPE|(?:RUNNING|TOTAL)_SCRIPT_COUNT|TOTAL_INVENTORY_COUNT|SCRIPT_(?:MEMORY|TIME))|TYPE_(?:INTEGER|FLOAT|STRING|KEY|VECTOR|ROTATION|INVALID)|(?:DEBUG|PUBLIC)_CHANNEL|ATTACH_(?:AVATAR_CENTER|CHEST|HEAD|BACK|PELVIS|MOUTH|CHIN|NECK|NOSE|BELLY|[LR](?:SHOULDER|HAND|FOOT|EAR|EYE|[UL](?:ARM|LEG)|HIP)|(?:LEFT|RIGHT)_PEC|HUD_(?:CENTER_[12]|TOP_(?:RIGHT|CENTER|LEFT)|BOTTOM(?:_(?:RIGHT|LEFT))?)|[LR]HAND_RING1|TAIL_(?:BASE|TIP)|[LR]WING|FACE_(?:JAW|[LR]EAR|[LR]EYE|TONGUE)|GROIN|HIND_[LR]FOOT)|LAND_(?:LEVEL|RAISE|LOWER|SMOOTH|NOISE|REVERT)|DATA_(?:ONLINE|NAME|BORN|SIM_(?:POS|STATUS|RATING)|PAYINFO)|PAYMENT_INFO_(?:ON_FILE|USED)|REMOTE_DATA_(?:CHANNEL|REQUEST|REPLY)|PSYS_(?:PART_(?:BF_(?:ZERO|ONE(?:_MINUS_(?:DEST_COLOR|SOURCE_(ALPHA|COLOR)))?|DEST_COLOR|SOURCE_(ALPHA|COLOR))|BLEND_FUNC_(DEST|SOURCE)|FLAGS|(?:START|END)_(?:COLOR|ALPHA|SCALE|GLOW)|MAX_AGE|(?:RIBBON|WIND|INTERP_(?:COLOR|SCALE)|BOUNCE|FOLLOW_(?:SRC|VELOCITY)|TARGET_(?:POS|LINEAR)|EMISSIVE)_MASK)|SRC_(?:MAX_AGE|PATTERN|ANGLE_(?:BEGIN|END)|BURST_(?:RATE|PART_COUNT|RADIUS|SPEED_(?:MIN|MAX))|ACCEL|TEXTURE|TARGET_KEY|OMEGA|PATTERN_(?:DROP|EXPLODE|ANGLE(?:_CONE(?:_EMPTY)?)?)))|VEHICLE_(?:REFERENCE_FRAME|TYPE_(?:NONE|SLED|CAR|BOAT|AIRPLANE|BALLOON)|(?:LINEAR|ANGULAR)_(?:FRICTION_TIMESCALE|MOTOR_DIRECTION)|LINEAR_MOTOR_OFFSET|HOVER_(?:HEIGHT|EFFICIENCY|TIMESCALE)|BUOYANCY|(?:LINEAR|ANGULAR)_(?:DEFLECTION_(?:EFFICIENCY|TIMESCALE)|MOTOR_(?:DECAY_)?TIMESCALE)|VERTICAL_ATTRACTION_(?:EFFICIENCY|TIMESCALE)|BANKING_(?:EFFICIENCY|MIX|TIMESCALE)|FLAG_(?:NO_DEFLECTION_UP|LIMIT_(?:ROLL_ONLY|MOTOR_UP)|HOVER_(?:(?:WATER|TERRAIN|UP)_ONLY|GLOBAL_HEIGHT)|MOUSELOOK_(?:STEER|BANK)|CAMERA_DECOUPLED))|PRIM_(?:ALPHA_MODE(?:_(?:BLEND|EMISSIVE|MASK|NONE))?|NORMAL|SPECULAR|TYPE(?:_(?:BOX|CYLINDER|PRISM|SPHERE|TORUS|TUBE|RING|SCULPT))?|HOLE_(?:DEFAULT|CIRCLE|SQUARE|TRIANGLE)|MATERIAL(?:_(?:STONE|METAL|GLASS|WOOD|FLESH|PLASTIC|RUBBER))?|SHINY_(?:NONE|LOW|MEDIUM|HIGH)|BUMP_(?:NONE|BRIGHT|DARK|WOOD|BARK|BRICKS|CHECKER|CONCRETE|TILE|STONE|DISKS|GRAVEL|BLOBS|SIDING|LARGETILE|STUCCO|SUCTION|WEAVE)|TEXGEN_(?:DEFAULT|PLANAR)|SCULPT_(?:TYPE_(?:SPHERE|TORUS|PLANE|CYLINDER|MASK)|FLAG_(?:MIRROR|INVERT))|PHYSICS(?:_(?:SHAPE_(?:CONVEX|NONE|PRIM|TYPE)))?|(?:POS|ROT)_LOCAL|SLICE|TEXT|FLEXIBLE|POINT_LIGHT|TEMP_ON_REZ|PHANTOM|POSITION|SIZE|ROTATION|TEXTURE|NAME|OMEGA|DESC|LINK_TARGET|COLOR|BUMP_SHINY|FULLBRIGHT|TEXGEN|GLOW|MEDIA_(?:ALT_IMAGE_ENABLE|CONTROLS|(?:CURRENT|HOME)_URL|AUTO_(?:LOOP|PLAY|SCALE|ZOOM)|FIRST_CLICK_INTERACT|(?:WIDTH|HEIGHT)_PIXELS|WHITELIST(?:_ENABLE)?|PERMS_(?:INTERACT|CONTROL)|PARAM_MAX|CONTROLS_(?:STANDARD|MINI)|PERM_(?:NONE|OWNER|GROUP|ANYONE)|MAX_(?:URL_LENGTH|WHITELIST_(?:SIZE|COUNT)|(?:WIDTH|HEIGHT)_PIXELS)))|MASK_(?:BASE|OWNER|GROUP|EVERYONE|NEXT)|PERM_(?:TRANSFER|MODIFY|COPY|MOVE|ALL)|PARCEL_(?:MEDIA_COMMAND_(?:STOP|PAUSE|PLAY|LOOP|TEXTURE|URL|TIME|AGENT|UNLOAD|AUTO_ALIGN|TYPE|SIZE|DESC|LOOP_SET)|FLAG_(?:ALLOW_(?:FLY|(?:GROUP_)?SCRIPTS|LANDMARK|TERRAFORM|DAMAGE|CREATE_(?:GROUP_)?OBJECTS)|USE_(?:ACCESS_(?:GROUP|LIST)|BAN_LIST|LAND_PASS_LIST)|LOCAL_SOUND_ONLY|RESTRICT_PUSHOBJECT|ALLOW_(?:GROUP|ALL)_OBJECT_ENTRY)|COUNT_(?:TOTAL|OWNER|GROUP|OTHER|SELECTED|TEMP)|DETAILS_(?:NAME|DESC|OWNER|GROUP|AREA|ID|SEE_AVATARS))|LIST_STAT_(?:MAX|MIN|MEAN|MEDIAN|STD_DEV|SUM(?:_SQUARES)?|NUM_COUNT|GEOMETRIC_MEAN|RANGE)|PAY_(?:HIDE|DEFAULT)|REGION_FLAG_(?:ALLOW_DAMAGE|FIXED_SUN|BLOCK_TERRAFORM|SANDBOX|DISABLE_(?:COLLISIONS|PHYSICS)|BLOCK_FLY|ALLOW_DIRECT_TELEPORT|RESTRICT_PUSHOBJECT)|HTTP_(?:METHOD|MIMETYPE|BODY_(?:MAXLENGTH|TRUNCATED)|CUSTOM_HEADER|PRAGMA_NO_CACHE|VERBOSE_THROTTLE|VERIFY_CERT)|STRING_(?:TRIM(?:_(?:HEAD|TAIL))?)|CLICK_ACTION_(?:NONE|TOUCH|SIT|BUY|PAY|OPEN(?:_MEDIA)?|PLAY|ZOOM)|TOUCH_INVALID_FACE|PROFILE_(?:NONE|SCRIPT_MEMORY)|RC_(?:DATA_FLAGS|DETECT_PHANTOM|GET_(?:LINK_NUM|NORMAL|ROOT_KEY)|MAX_HITS|REJECT_(?:TYPES|AGENTS|(?:NON)?PHYSICAL|LAND))|RCERR_(?:CAST_TIME_EXCEEDED|SIM_PERF_LOW|UNKNOWN)|ESTATE_ACCESS_(?:ALLOWED_(?:AGENT|GROUP)_(?:ADD|REMOVE)|BANNED_AGENT_(?:ADD|REMOVE))|DENSITY|FRICTION|RESTITUTION|GRAVITY_MULTIPLIER|KFM_(?:COMMAND|CMD_(?:PLAY|STOP|PAUSE)|MODE|FORWARD|LOOP|PING_PONG|REVERSE|DATA|ROTATION|TRANSLATION)|ERR_(?:GENERIC|PARCEL_PERMISSIONS|MALFORMED_PARAMS|RUNTIME_PERMISSIONS|THROTTLED)|CHARACTER_(?:CMD_(?:(?:SMOOTH_)?STOP|JUMP)|DESIRED_(?:TURN_)?SPEED|RADIUS|STAY_WITHIN_PARCEL|LENGTH|ORIENTATION|ACCOUNT_FOR_SKIPPED_FRAMES|AVOIDANCE_MODE|TYPE(?:_(?:[ABCD]|NONE))?|MAX_(?:DECEL|TURN_RADIUS|(?:ACCEL|SPEED)))|PURSUIT_(?:OFFSET|FUZZ_FACTOR|GOAL_TOLERANCE|INTERCEPT)|REQUIRE_LINE_OF_SIGHT|FORCE_DIRECT_PATH|VERTICAL|HORIZONTAL|AVOID_(?:CHARACTERS|DYNAMIC_OBSTACLES|NONE)|PU_(?:EVADE_(?:HIDDEN|SPOTTED)|FAILURE_(?:DYNAMIC_PATHFINDING_DISABLED|INVALID_(?:GOAL|START)|NO_(?:NAVMESH|VALID_DESTINATION)|OTHER|TARGET_GONE|(?:PARCEL_)?UNREACHABLE)|(?:GOAL|SLOWDOWN_DISTANCE)_REACHED)|TRAVERSAL_TYPE(?:_(?:FAST|NONE|SLOW))?|CONTENT_TYPE_(?:ATOM|FORM|HTML|JSON|LLSD|RSS|TEXT|XHTML|XML)|GCNP_(?:RADIUS|STATIC)|(?:PATROL|WANDER)_PAUSE_AT_WAYPOINTS|OPT_(?:AVATAR|CHARACTER|EXCLUSION_VOLUME|LEGACY_LINKSET|MATERIAL_VOLUME|OTHER|STATIC_OBSTACLE|WALKABLE)|SIM_STAT_PCT_CHARS_STEPPED)\\b"},{b:"\\b(?:FALSE|TRUE)\\b"},{b:"\\b(?:ZERO_ROTATION)\\b"},{b:"\\b(?:EOF|JSON_(?:ARRAY|DELETE|FALSE|INVALID|NULL|NUMBER|OBJECT|STRING|TRUE)|NULL_KEY|TEXTURE_(?:BLANK|DEFAULT|MEDIA|PLYWOOD|TRANSPARENT)|URL_REQUEST_(?:GRANTED|DENIED))\\b"},{b:"\\b(?:ZERO_VECTOR|TOUCH_INVALID_(?:TEXCOORD|VECTOR))\\b"}]},n={cN:"built_in",b:"\\b(?:ll(?:AgentInExperience|(?:Create|DataSize|Delete|KeyCount|Keys|Read|Update)KeyValue|GetExperience(?:Details|ErrorMessage)|ReturnObjectsBy(?:ID|Owner)|Json(?:2List|[GS]etValue|ValueType)|Sin|Cos|Tan|Atan2|Sqrt|Pow|Abs|Fabs|Frand|Floor|Ceil|Round|Vec(?:Mag|Norm|Dist)|Rot(?:Between|2(?:Euler|Fwd|Left|Up))|(?:Euler|Axes)2Rot|Whisper|(?:Region|Owner)?Say|Shout|Listen(?:Control|Remove)?|Sensor(?:Repeat|Remove)?|Detected(?:Name|Key|Owner|Type|Pos|Vel|Grab|Rot|Group|LinkNumber)|Die|Ground|Wind|(?:[GS]et)(?:AnimationOverride|MemoryLimit|PrimMediaParams|ParcelMusicURL|Object(?:Desc|Name)|PhysicsMaterial|Status|Scale|Color|Alpha|Texture|Pos|Rot|Force|Torque)|ResetAnimationOverride|(?:Scale|Offset|Rotate)Texture|(?:Rot)?Target(?:Remove)?|(?:Stop)?MoveToTarget|Apply(?:Rotational)?Impulse|Set(?:KeyframedMotion|ContentType|RegionPos|(?:Angular)?Velocity|Buoyancy|HoverHeight|ForceAndTorque|TimerEvent|ScriptState|Damage|TextureAnim|Sound(?:Queueing|Radius)|Vehicle(?:Type|(?:Float|Vector|Rotation)Param)|(?:Touch|Sit)?Text|Camera(?:Eye|At)Offset|PrimitiveParams|ClickAction|Link(?:Alpha|Color|PrimitiveParams(?:Fast)?|Texture(?:Anim)?|Camera|Media)|RemoteScriptAccessPin|PayPrice|LocalRot)|ScaleByFactor|Get(?:(?:Max|Min)ScaleFactor|ClosestNavPoint|StaticPath|SimStats|Env|PrimitiveParams|Link(?:PrimitiveParams|Number(?:OfSides)?|Key|Name|Media)|HTTPHeader|FreeURLs|Object(?:Details|PermMask|PrimCount)|Parcel(?:MaxPrims|Details|Prim(?:Count|Owners))|Attached(?:List)?|(?:SPMax|Free|Used)Memory|Region(?:Name|TimeDilation|FPS|Corner|AgentCount)|Root(?:Position|Rotation)|UnixTime|(?:Parcel|Region)Flags|(?:Wall|GMT)clock|SimulatorHostname|BoundingBox|GeometricCenter|Creator|NumberOf(?:Prims|NotecardLines|Sides)|Animation(?:List)?|(?:Camera|Local)(?:Pos|Rot)|Vel|Accel|Omega|Time(?:stamp|OfDay)|(?:Object|CenterOf)?Mass|MassMKS|Energy|Owner|(?:Owner)?Key|SunDirection|Texture(?:Offset|Scale|Rot)|Inventory(?:Number|Name|Key|Type|Creator|PermMask)|Permissions(?:Key)?|StartParameter|List(?:Length|EntryType)|Date|Agent(?:Size|Info|Language|List)|LandOwnerAt|NotecardLine|Script(?:Name|State))|(?:Get|Reset|GetAndReset)Time|PlaySound(?:Slave)?|LoopSound(?:Master|Slave)?|(?:Trigger|Stop|Preload)Sound|(?:(?:Get|Delete)Sub|Insert)String|To(?:Upper|Lower)|Give(?:InventoryList|Money)|RezObject|(?:Stop)?LookAt|Sleep|CollisionFilter|(?:Take|Release)Controls|DetachFromAvatar|AttachToAvatar(?:Temp)?|InstantMessage|(?:GetNext)?Email|StopHover|MinEventDelay|RotLookAt|String(?:Length|Trim)|(?:Start|Stop)Animation|TargetOmega|Request(?:Experience)?Permissions|(?:Create|Break)Link|BreakAllLinks|(?:Give|Remove)Inventory|Water|PassTouches|Request(?:Agent|Inventory)Data|TeleportAgent(?:Home|GlobalCoords)?|ModifyLand|CollisionSound|ResetScript|MessageLinked|PushObject|PassCollisions|AxisAngle2Rot|Rot2(?:Axis|Angle)|A(?:cos|sin)|AngleBetween|AllowInventoryDrop|SubStringIndex|List2(?:CSV|Integer|Json|Float|String|Key|Vector|Rot|List(?:Strided)?)|DeleteSubList|List(?:Statistics|Sort|Randomize|(?:Insert|Find|Replace)List)|EdgeOfWorld|AdjustSoundVolume|Key2Name|TriggerSoundLimited|EjectFromLand|(?:CSV|ParseString)2List|OverMyLand|SameGroup|UnSit|Ground(?:Slope|Normal|Contour)|GroundRepel|(?:Set|Remove)VehicleFlags|(?:AvatarOn)?(?:Link)?SitTarget|Script(?:Danger|Profiler)|Dialog|VolumeDetect|ResetOtherScript|RemoteLoadScriptPin|(?:Open|Close)RemoteDataChannel|SendRemoteData|RemoteDataReply|(?:Integer|String)ToBase64|XorBase64|Log(?:10)?|Base64To(?:String|Integer)|ParseStringKeepNulls|RezAtRoot|RequestSimulatorData|ForceMouselook|(?:Load|Release|(?:E|Une)scape)URL|ParcelMedia(?:CommandList|Query)|ModPow|MapDestination|(?:RemoveFrom|AddTo|Reset)Land(?:Pass|Ban)List|(?:Set|Clear)CameraParams|HTTP(?:Request|Response)|TextBox|DetectedTouch(?:UV|Face|Pos|(?:N|Bin)ormal|ST)|(?:MD5|SHA1|DumpList2)String|Request(?:Secure)?URL|Clear(?:Prim|Link)Media|(?:Link)?ParticleSystem|(?:Get|Request)(?:Username|DisplayName)|RegionSayTo|CastRay|GenerateKey|TransferLindenDollars|ManageEstateAccess|(?:Create|Delete)Character|ExecCharacterCmd|Evade|FleeFrom|NavigateTo|PatrolPoints|Pursue|UpdateCharacter|WanderWithin))\\b"};return{i:":",c:[r,{cN:"comment",v:[e.C("//","$"),e.C("/\\*","\\*/")]},a,{cN:"section",v:[{b:"\\b(?:state|default)\\b"},{b:"\\b(?:state_(?:entry|exit)|touch(?:_(?:start|end))?|(?:land_)?collision(?:_(?:start|end))?|timer|listen|(?:no_)?sensor|control|(?:not_)?at_(?:rot_)?target|money|email|experience_permissions(?:_denied)?|run_time_permissions|changed|attach|dataserver|moving_(?:start|end)|link_message|(?:on|object)_rez|remote_data|http_re(?:sponse|quest)|path_update|transaction_result)\\b"}]},n,i,{cN:"type",b:"\\b(?:integer|float|string|key|vector|quaternion|rotation|list)\\b"}]}})),e.registerLanguage("lua",(function(e){var t="\\[=*\\[",r="\\]=*\\]",a={b:t,e:r,c:["self"]},i=[e.C("--(?!"+t+")","$"),e.C("--"+t,r,{c:[a],r:10})];return{l:e.UIR,k:{literal:"true false nil",keyword:"and break do else elseif end for goto if in local not or repeat return then until while",built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstringmodule next pairs pcall print rawequal rawget rawset require select setfenvsetmetatable tonumber tostring type unpack xpcall arg selfcoroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"},c:i.concat([{cN:"function",bK:"function",e:"\\)",c:[e.inherit(e.TM,{b:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{cN:"params",b:"\\(",eW:!0,c:i}].concat(i)},e.CNM,e.ASM,e.QSM,{cN:"string",b:t,e:r,c:[a],r:5}])}})),e.registerLanguage("makefile",(function(e){var t={cN:"variable",v:[{b:"\\$\\("+e.UIR+"\\)",c:[e.BE]},{b:/\$[@%"},{b:"<=",r:0},{b:"=>",r:0},{b:"/\\\\"},{b:"\\\\/"}]},l={cN:"built_in",v:[{b:":-\\|--\x3e"},{b:"=",r:0}]};return{aliases:["m","moo"],k:t,c:[s,l,r,e.CBCM,a,e.NM,i,n,{b:/:-/}]}})),e.registerLanguage("mipsasm",(function(e){return{cI:!0,aliases:["mips"],l:"\\.?"+e.IR,k:{meta:".2byte .4byte .align .ascii .asciz .balign .byte .code .data .else .end .endif .endm .endr .equ .err .exitm .extern .global .hword .if .ifdef .ifndef .include .irp .long .macro .rept .req .section .set .skip .space .text .word .ltorg ",built_in:"$0 $1 $2 $3 $4 $5 $6 $7 $8 $9 $10 $11 $12 $13 $14 $15 $16 $17 $18 $19 $20 $21 $22 $23 $24 $25 $26 $27 $28 $29 $30 $31 zero at v0 v1 a0 a1 a2 a3 a4 a5 a6 a7 t0 t1 t2 t3 t4 t5 t6 t7 t8 t9 s0 s1 s2 s3 s4 s5 s6 s7 s8 k0 k1 gp sp fp ra $f0 $f1 $f2 $f2 $f4 $f5 $f6 $f7 $f8 $f9 $f10 $f11 $f12 $f13 $f14 $f15 $f16 $f17 $f18 $f19 $f20 $f21 $f22 $f23 $f24 $f25 $f26 $f27 $f28 $f29 $f30 $f31 Context Random EntryLo0 EntryLo1 Context PageMask Wired EntryHi HWREna BadVAddr Count Compare SR IntCtl SRSCtl SRSMap Cause EPC PRId EBase Config Config1 Config2 Config3 LLAddr Debug DEPC DESAVE CacheErr ECC ErrorEPC TagLo DataLo TagHi DataHi WatchLo WatchHi PerfCtl PerfCnt "},c:[{cN:"keyword",b:"\\b(addi?u?|andi?|b(al)?|beql?|bgez(al)?l?|bgtzl?|blezl?|bltz(al)?l?|bnel?|cl[oz]|divu?|ext|ins|j(al)?|jalr(.hb)?|jr(.hb)?|lbu?|lhu?|ll|lui|lw[lr]?|maddu?|mfhi|mflo|movn|movz|move|msubu?|mthi|mtlo|mul|multu?|nop|nor|ori?|rotrv?|sb|sc|se[bh]|sh|sllv?|slti?u?|srav?|srlv?|subu?|sw[lr]?|xori?|wsbh|abs.[sd]|add.[sd]|alnv.ps|bc1[ft]l?|c.(s?f|un|u?eq|[ou]lt|[ou]le|ngle?|seq|l[et]|ng[et]).[sd]|(ceil|floor|round|trunc).[lw].[sd]|cfc1|cvt.d.[lsw]|cvt.l.[dsw]|cvt.ps.s|cvt.s.[dlw]|cvt.s.p[lu]|cvt.w.[dls]|div.[ds]|ldx?c1|luxc1|lwx?c1|madd.[sd]|mfc1|mov[fntz]?.[ds]|msub.[sd]|mth?c1|mul.[ds]|neg.[ds]|nmadd.[ds]|nmsub.[ds]|p[lu][lu].ps|recip.fmt|r?sqrt.[ds]|sdx?c1|sub.[ds]|suxc1|swx?c1|break|cache|d?eret|[de]i|ehb|mfc0|mtc0|pause|prefx?|rdhwr|rdpgpr|sdbbp|ssnop|synci?|syscall|teqi?|tgei?u?|tlb(p|r|w[ir])|tlti?u?|tnei?|wait|wrpgpr)",e:"\\s"},e.C("[;#]","$"),e.CBCM,e.QSM,{cN:"string",b:"'",e:"[^\\\\]'",r:0},{cN:"title",b:"\\|",e:"\\|",i:"\\n",r:0},{cN:"number",v:[{b:"0x[0-9a-f]+"},{b:"\\b-?\\d+"}],r:0},{cN:"symbol",v:[{b:"^\\s*[a-z_\\.\\$][a-z0-9_\\.\\$]+:"},{b:"^\\s*[0-9]+:"},{b:"[0-9]+[bf]"}],r:0}],i:"/"}})),e.registerLanguage("mizar",(function(e){return{k:"environ vocabularies notations constructors definitions registrations theorems schemes requirements begin end definition registration cluster existence pred func defpred deffunc theorem proof let take assume then thus hence ex for st holds consider reconsider such that and in provided of as from be being by means equals implies iff redefine define now not or attr is mode suppose per cases set thesis contradiction scheme reserve struct correctness compatibility coherence symmetry assymetry reflexivity irreflexivity connectedness uniqueness commutativity idempotence involutiveness projectivity",c:[e.C("::","$")]}})),e.registerLanguage("perl",(function(e){var t="getpwent getservent quotemeta msgrcv scalar kill dbmclose undef lc ma syswrite tr send umask sysopen shmwrite vec qx utime local oct semctl localtime readpipe do return format read sprintf dbmopen pop getpgrp not getpwnam rewinddir qqfileno qw endprotoent wait sethostent bless s|0 opendir continue each sleep endgrent shutdown dump chomp connect getsockname die socketpair close flock exists index shmgetsub for endpwent redo lstat msgctl setpgrp abs exit select print ref gethostbyaddr unshift fcntl syscall goto getnetbyaddr join gmtime symlink semget splice x|0 getpeername recv log setsockopt cos last reverse gethostbyname getgrnam study formline endhostent times chop length gethostent getnetent pack getprotoent getservbyname rand mkdir pos chmod y|0 substr endnetent printf next open msgsnd readdir use unlink getsockopt getpriority rindex wantarray hex system getservbyport endservent int chr untie rmdir prototype tell listen fork shmread ucfirst setprotoent else sysseek link getgrgid shmctl waitpid unpack getnetbyname reset chdir grep split require caller lcfirst until warn while values shift telldir getpwuid my getprotobynumber delete and sort uc defined srand accept package seekdir getprotobyname semop our rename seek if q|0 chroot sysread setpwent no crypt getc chown sqrt write setnetent setpriority foreach tie sin msgget map stat getlogin unless elsif truncate exec keys glob tied closedirioctl socket readlink eval xor readline binmode setservent eof ord bind alarm pipe atan2 getgrent exp time push setgrent gt lt or ne m|0 break given say state when",r={cN:"subst",b:"[$@]\\{",e:"\\}",k:t},a={b:"->{",e:"}"},i={v:[{b:/\$\d/},{b:/[\$%@](\^\w\b|#\w+(::\w+)*|{\w+}|\w+(::\w*)*)/},{b:/[\$%@][^\s\w{]/,r:0}]},n=[e.BE,r,i],o=[i,e.HCM,e.C("^\\=\\w","\\=cut",{eW:!0}),a,{cN:"string",c:n,v:[{b:"q[qwxr]?\\s*\\(",e:"\\)",r:5},{b:"q[qwxr]?\\s*\\[",e:"\\]",r:5},{b:"q[qwxr]?\\s*\\{",e:"\\}",r:5},{b:"q[qwxr]?\\s*\\|",e:"\\|",r:5},{b:"q[qwxr]?\\s*\\<",e:"\\>",r:5},{b:"qw\\s+q",e:"q",r:5},{b:"'",e:"'",c:[e.BE]},{b:'"',e:'"'},{b:"`",e:"`",c:[e.BE]},{b:"{\\w+}",c:[],r:0},{b:"-?\\w+\\s*\\=\\>",c:[],r:0}]},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{b:"(\\/\\/|"+e.RSR+"|\\b(split|return|print|reverse|grep)\\b)\\s*",k:"split return print reverse grep",r:0,c:[e.HCM,{cN:"regexp",b:"(s|tr|y)/(\\\\.|[^/])*/(\\\\.|[^/])*/[a-z]*",r:10},{cN:"regexp",b:"(m|qr)?/",e:"/[a-z]*",c:[e.BE],r:0}]},{cN:"function",bK:"sub",e:"(\\s*\\(.*?\\))?[;{]",eE:!0,r:5,c:[e.TM]},{b:"-\\w\\b",r:0},{b:"^__DATA__$",e:"^__END__$",sL:"mojolicious",c:[{b:"^@@.*",e:"$",cN:"comment"}]}];return r.c=o,a.c=o,{aliases:["pl","pm"],l:/[\w\.]+/,k:t,c:o}})),e.registerLanguage("mojolicious",(function(e){return{sL:"xml",c:[{cN:"meta",b:"^__(END|DATA)__$"},{b:"^\\s*%{1,2}={0,2}",e:"$",sL:"perl"},{b:"<%{1,2}={0,2}",e:"={0,1}%>",sL:"perl",eB:!0,eE:!0}]}})),e.registerLanguage("monkey",(function(e){var t={cN:"number",r:0,v:[{b:"[$][a-fA-F0-9]+"},e.NM]};return{cI:!0,k:{keyword:"public private property continue exit extern new try catch eachin not abstract final select case default const local global field end if then else elseif endif while wend repeat until forever for to step next return module inline throw import",built_in:"DebugLog DebugStop Error Print ACos ACosr ASin ASinr ATan ATan2 ATan2r ATanr Abs Abs Ceil Clamp Clamp Cos Cosr Exp Floor Log Max Max Min Min Pow Sgn Sgn Sin Sinr Sqrt Tan Tanr Seed PI HALFPI TWOPI",literal:"true false null and or shl shr mod"},i:/\/\*/,c:[e.C("#rem","#end"),e.C("'","$",{r:0}),{cN:"function",bK:"function method",e:"[(=:]|$",i:/\n/,c:[e.UTM]},{cN:"class",bK:"class interface",e:"$",c:[{bK:"extends implements"},e.UTM]},{cN:"built_in",b:"\\b(self|super)\\b"},{cN:"meta",b:"\\s*#",e:"$",k:{"meta-keyword":"if else elseif endif end then"}},{cN:"meta",b:"^\\s*strict\\b"},{bK:"alias",e:"=",c:[e.UTM]},e.QSM,t]}})),e.registerLanguage("moonscript",(function(e){var t={keyword:"if then not for in while do return else elseif break continue switch and or unless when class extends super local import export from using",literal:"true false nil",built_in:"_G _VERSION assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall coroutine debug io math os package string table"},r="[A-Za-z$_][0-9A-Za-z$_]*",a={cN:"subst",b:/#\{/,e:/}/,k:t},i=[e.inherit(e.CNM,{starts:{e:"(\\s*/)?",r:0}}),{cN:"string",v:[{b:/'/,e:/'/,c:[e.BE]},{b:/"/,e:/"/,c:[e.BE,a]}]},{cN:"built_in",b:"@__"+e.IR},{b:"@"+e.IR},{b:e.IR+"\\\\"+e.IR}];a.c=i;var n=e.inherit(e.TM,{b:r}),o="(\\(.*\\))?\\s*\\B[-=]>",s={cN:"params",b:"\\([^\\(]",rB:!0,c:[{b:/\(/,e:/\)/,k:t,c:["self"].concat(i)}]};return{aliases:["moon"],k:t,i:/\/\*/,c:i.concat([e.C("--","$"),{cN:"function",b:"^\\s*"+r+"\\s*=\\s*"+o,e:"[-=]>",rB:!0,c:[n,s]},{b:/[\(,:=]\s*/,r:0,c:[{cN:"function",b:o,e:"[-=]>",rB:!0,c:[s]}]},{cN:"class",bK:"class",e:"$",i:/[:="\[\]]/,c:[{bK:"extends",eW:!0,i:/[:="\[\]]/,c:[n]},n]},{cN:"name",b:r+":",e:":",rB:!0,rE:!0,r:0}])}})),e.registerLanguage("n1ql",(function(e){return{cI:!0,c:[{bK:"build create index delete drop explain infer|10 insert merge prepare select update upsert|10",e:/;/,eW:!0,k:{keyword:"all alter analyze and any array as asc begin between binary boolean break bucket build by call case cast cluster collate collection commit connect continue correlate cover create database dataset datastore declare decrement delete derived desc describe distinct do drop each element else end every except exclude execute exists explain fetch first flatten for force from function grant group gsi having if ignore ilike in include increment index infer inline inner insert intersect into is join key keys keyspace known last left let letting like limit lsm map mapping matched materialized merge minus namespace nest not number object offset on option or order outer over parse partition password path pool prepare primary private privilege procedure public raw realm reduce rename return returning revoke right role rollback satisfies schema select self semi set show some start statistics string system then to transaction trigger truncate under union unique unknown unnest unset update upsert use user using validate value valued values via view when where while with within work xor",literal:"true false null missing|5",built_in:"array_agg array_append array_concat array_contains array_count array_distinct array_ifnull array_length array_max array_min array_position array_prepend array_put array_range array_remove array_repeat array_replace array_reverse array_sort array_sum avg count max min sum greatest least ifmissing ifmissingornull ifnull missingif nullif ifinf ifnan ifnanorinf naninf neginfif posinfif clock_millis clock_str date_add_millis date_add_str date_diff_millis date_diff_str date_part_millis date_part_str date_trunc_millis date_trunc_str duration_to_str millis str_to_millis millis_to_str millis_to_utc millis_to_zone_name now_millis now_str str_to_duration str_to_utc str_to_zone_name decode_json encode_json encoded_size poly_length base64 base64_encode base64_decode meta uuid abs acos asin atan atan2 ceil cos degrees e exp ln log floor pi power radians random round sign sin sqrt tan trunc object_length object_names object_pairs object_inner_pairs object_values object_inner_values object_add object_put object_remove object_unwrap regexp_contains regexp_like regexp_position regexp_replace contains initcap length lower ltrim position repeat replace rtrim split substr title trim upper isarray isatom isboolean isnumber isobject isstring type toarray toatom toboolean tonumber toobject tostring"},c:[{cN:"string",b:"'",e:"'",c:[e.BE],r:0},{cN:"string",b:'"',e:'"',c:[e.BE],r:0},{cN:"symbol",b:"`",e:"`",c:[e.BE],r:2},e.CNM,e.CBCM]},e.CBCM]}})),e.registerLanguage("nginx",(function(e){var t={cN:"variable",v:[{b:/\$\d+/},{b:/\$\{/,e:/}/},{b:"[\\$\\@]"+e.UIR}]},r={eW:!0,l:"[a-z/_]+",k:{literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll"},r:0,i:"=>",c:[e.HCM,{cN:"string",c:[e.BE,t],v:[{b:/"/,e:/"/},{b:/'/,e:/'/}]},{b:"([a-z]+):/",e:"\\s",eW:!0,eE:!0,c:[t]},{cN:"regexp",c:[e.BE,t],v:[{b:"\\s\\^",e:"\\s|{|;",rE:!0},{b:"~\\*?\\s+",e:"\\s|{|;",rE:!0},{b:"\\*(\\.[a-z\\-]+)+"},{b:"([a-z\\-]+\\.)+\\*"}]},{cN:"number",b:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{cN:"number",b:"\\b\\d+[kKmMgGdshdwy]*\\b",r:0},t]};return{aliases:["nginxconf"],c:[e.HCM,{b:e.UIR+"\\s+{",rB:!0,e:"{",c:[{cN:"section",b:e.UIR}],r:0},{b:e.UIR+"\\s",e:";|{",rB:!0,c:[{cN:"attribute",b:e.UIR,starts:r}],r:0}],i:"[^\\s\\}]"}})),e.registerLanguage("nimrod",(function(e){return{aliases:["nim"],k:{keyword:"addr and as asm bind block break case cast const continue converter discard distinct div do elif else end enum except export finally for from generic if import in include interface is isnot iterator let macro method mixin mod nil not notin object of or out proc ptr raise ref return shl shr static template try tuple type using var when while with without xor yield",literal:"shared guarded stdin stdout stderr result true false",built_in:"int int8 int16 int32 int64 uint uint8 uint16 uint32 uint64 float float32 float64 bool char string cstring pointer expr stmt void auto any range array openarray varargs seq set clong culong cchar cschar cshort cint csize clonglong cfloat cdouble clongdouble cuchar cushort cuint culonglong cstringarray semistatic"},c:[{cN:"meta",b:/{\./,e:/\.}/,r:10},{cN:"string",b:/[a-zA-Z]\w*"/,e:/"/,c:[{b:/""/}]},{cN:"string",b:/([a-zA-Z]\w*)?"""/,e:/"""/},e.QSM,{cN:"type",b:/\b[A-Z]\w+\b/,r:0},{cN:"number",r:0,v:[{b:/\b(0[xX][0-9a-fA-F][_0-9a-fA-F]*)('?[iIuU](8|16|32|64))?/},{b:/\b(0o[0-7][_0-7]*)('?[iIuUfF](8|16|32|64))?/},{b:/\b(0(b|B)[01][_01]*)('?[iIuUfF](8|16|32|64))?/},{b:/\b(\d[_\d]*)('?[iIuUfF](8|16|32|64))?/}]},e.HCM]}})),e.registerLanguage("nix",(function(e){var t={keyword:"rec with let in inherit assert if else then",literal:"true false or and null",built_in:"import abort baseNameOf dirOf isNull builtins map removeAttrs throw toString derivation"},r={cN:"subst",b:/\$\{/,e:/}/,k:t},a={b:/[a-zA-Z0-9-_]+(\s*=)/,rB:!0,r:0,c:[{cN:"attr",b:/\S+/}]},i={cN:"string",c:[r],v:[{b:"''",e:"''"},{b:'"',e:'"'}]},n=[e.NM,e.HCM,e.CBCM,i,a];return r.c=n,{aliases:["nixos"],k:t,c:n}})),e.registerLanguage("nsis",(function(e){var t={cN:"variable",b:/\$(ADMINTOOLS|APPDATA|CDBURN_AREA|CMDLINE|COMMONFILES32|COMMONFILES64|COMMONFILES|COOKIES|DESKTOP|DOCUMENTS|EXEDIR|EXEFILE|EXEPATH|FAVORITES|FONTS|HISTORY|HWNDPARENT|INSTDIR|INTERNET_CACHE|LANGUAGE|LOCALAPPDATA|MUSIC|NETHOOD|OUTDIR|PICTURES|PLUGINSDIR|PRINTHOOD|PROFILE|PROGRAMFILES32|PROGRAMFILES64|PROGRAMFILES|QUICKLAUNCH|RECENT|RESOURCES_LOCALIZED|RESOURCES|SENDTO|SMPROGRAMS|SMSTARTUP|STARTMENU|SYSDIR|TEMP|TEMPLATES|VIDEOS|WINDIR)/},r={cN:"variable",b:/\$+{[\w\.:-]+}/},a={cN:"variable",b:/\$+\w+/,i:/\(\){}/},i={cN:"variable",b:/\$+\([\w\^\.:-]+\)/},n={cN:"params",b:"(ARCHIVE|FILE_ATTRIBUTE_ARCHIVE|FILE_ATTRIBUTE_NORMAL|FILE_ATTRIBUTE_OFFLINE|FILE_ATTRIBUTE_READONLY|FILE_ATTRIBUTE_SYSTEM|FILE_ATTRIBUTE_TEMPORARY|HKCR|HKCU|HKDD|HKEY_CLASSES_ROOT|HKEY_CURRENT_CONFIG|HKEY_CURRENT_USER|HKEY_DYN_DATA|HKEY_LOCAL_MACHINE|HKEY_PERFORMANCE_DATA|HKEY_USERS|HKLM|HKPD|HKU|IDABORT|IDCANCEL|IDIGNORE|IDNO|IDOK|IDRETRY|IDYES|MB_ABORTRETRYIGNORE|MB_DEFBUTTON1|MB_DEFBUTTON2|MB_DEFBUTTON3|MB_DEFBUTTON4|MB_ICONEXCLAMATION|MB_ICONINFORMATION|MB_ICONQUESTION|MB_ICONSTOP|MB_OK|MB_OKCANCEL|MB_RETRYCANCEL|MB_RIGHT|MB_RTLREADING|MB_SETFOREGROUND|MB_TOPMOST|MB_USERICON|MB_YESNO|NORMAL|OFFLINE|READONLY|SHCTX|SHELL_CONTEXT|SYSTEM|TEMPORARY)"},o={cN:"keyword",b:/\!(addincludedir|addplugindir|appendfile|cd|define|delfile|echo|else|endif|error|execute|finalize|getdllversionsystem|ifdef|ifmacrodef|ifmacrondef|ifndef|if|include|insertmacro|macroend|macro|makensis|packhdr|searchparse|searchreplace|tempfile|undef|verbose|warning)/},s={cN:"subst",b:/\$(\\[nrt]|\$)/},l={cN:"class",b:/\w+\:\:\w+/},c={cN:"string",v:[{b:'"',e:'"'},{b:"'",e:"'"},{b:"`",e:"`"}],i:/\n/,c:[s,t,r,a,i]};return{cI:!1,k:{keyword:"Abort AddBrandingImage AddSize AllowRootDirInstall AllowSkipFiles AutoCloseWindow BGFont BGGradient BrandingText BringToFront Call CallInstDLL Caption ChangeUI CheckBitmap ClearErrors CompletedText ComponentText CopyFiles CRCCheck CreateDirectory CreateFont CreateShortCut Delete DeleteINISec DeleteINIStr DeleteRegKey DeleteRegValue DetailPrint DetailsButtonText DirText DirVar DirVerify EnableWindow EnumRegKey EnumRegValue Exch Exec ExecShell ExecWait ExpandEnvStrings File FileBufSize FileClose FileErrorText FileOpen FileRead FileReadByte FileReadUTF16LE FileReadWord FileSeek FileWrite FileWriteByte FileWriteUTF16LE FileWriteWord FindClose FindFirst FindNext FindWindow FlushINI FunctionEnd GetCurInstType GetCurrentAddress GetDlgItem GetDLLVersion GetDLLVersionLocal GetErrorLevel GetFileTime GetFileTimeLocal GetFullPathName GetFunctionAddress GetInstDirError GetLabelAddress GetTempFileName Goto HideWindow Icon IfAbort IfErrors IfFileExists IfRebootFlag IfSilent InitPluginsDir InstallButtonText InstallColors InstallDir InstallDirRegKey InstProgressFlags InstType InstTypeGetText InstTypeSetText IntCmp IntCmpU IntFmt IntOp IsWindow LangString LicenseBkColor LicenseData LicenseForceSelection LicenseLangString LicenseText LoadLanguageFile LockWindow LogSet LogText ManifestDPIAware ManifestSupportedOS MessageBox MiscButtonText Name Nop OutFile Page PageCallbacks PageExEnd Pop Push Quit ReadEnvStr ReadINIStr ReadRegDWORD ReadRegStr Reboot RegDLL Rename RequestExecutionLevel ReserveFile Return RMDir SearchPath SectionEnd SectionGetFlags SectionGetInstTypes SectionGetSize SectionGetText SectionGroupEnd SectionIn SectionSetFlags SectionSetInstTypes SectionSetSize SectionSetText SendMessage SetAutoClose SetBrandingImage SetCompress SetCompressor SetCompressorDictSize SetCtlColors SetCurInstType SetDatablockOptimize SetDateSave SetDetailsPrint SetDetailsView SetErrorLevel SetErrors SetFileAttributes SetFont SetOutPath SetOverwrite SetRebootFlag SetRegView SetShellVarContext SetSilent ShowInstDetails ShowUninstDetails ShowWindow SilentInstall SilentUnInstall Sleep SpaceTexts StrCmp StrCmpS StrCpy StrLen SubCaption Unicode UninstallButtonText UninstallCaption UninstallIcon UninstallSubCaption UninstallText UninstPage UnRegDLL Var VIAddVersionKey VIFileVersion VIProductVersion WindowIcon WriteINIStr WriteRegBin WriteRegDWORD WriteRegExpandStr WriteRegStr WriteUninstaller XPStyle",literal:"admin all auto both bottom bzip2 colored components current custom directory false force hide highest ifdiff ifnewer instfiles lastused leave left license listonly lzma nevershow none normal notset off on open print right show silent silentlog smooth textonly top true try un.components un.custom un.directory un.instfiles un.license uninstConfirm user Win10 Win7 Win8 WinVista zlib"},c:[e.HCM,e.CBCM,e.C(";","$",{r:0}),{cN:"function",bK:"Function PageEx Section SectionGroup",e:"$"},c,o,r,a,i,n,l,e.NM]}})),e.registerLanguage("objectivec",(function(e){var t={cN:"built_in",b:"\\b(AV|CA|CF|CG|CI|CL|CM|CN|CT|MK|MP|MTK|MTL|NS|SCN|SK|UI|WK|XC)\\w+"},r={keyword:"int float while char export sizeof typedef const struct for union unsigned long volatile static bool mutable if do return goto void enum else break extern asm case short default double register explicit signed typename this switch continue wchar_t inline readonly assign readwrite self @synchronized id typeof nonatomic super unichar IBOutlet IBAction strong weak copy in out inout bycopy byref oneway __strong __weak __block __autoreleasing @private @protected @public @try @property @end @throw @catch @finally @autoreleasepool @synthesize @dynamic @selector @optional @required @encode @package @import @defs @compatibility_alias __bridge __bridge_transfer __bridge_retained __bridge_retain __covariant __contravariant __kindof _Nonnull _Nullable _Null_unspecified __FUNCTION__ __PRETTY_FUNCTION__ __attribute__ getter setter retain unsafe_unretained nonnull nullable null_unspecified null_resettable class instancetype NS_DESIGNATED_INITIALIZER NS_UNAVAILABLE NS_REQUIRES_SUPER NS_RETURNS_INNER_POINTER NS_INLINE NS_AVAILABLE NS_DEPRECATED NS_ENUM NS_OPTIONS NS_SWIFT_UNAVAILABLE NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_END NS_REFINED_FOR_SWIFT NS_SWIFT_NAME NS_SWIFT_NOTHROW NS_DURING NS_HANDLER NS_ENDHANDLER NS_VALUERETURN NS_VOIDRETURN",literal:"false true FALSE TRUE nil YES NO NULL",built_in:"BOOL dispatch_once_t dispatch_queue_t dispatch_sync dispatch_async dispatch_once"},a=/[a-zA-Z@][a-zA-Z0-9_]*/,i="@interface @class @protocol @implementation";return{aliases:["mm","objc","obj-c"],k:r,l:a,i:""}]}]},{cN:"class",b:"("+i.split(" ").join("|")+")\\b",e:"({|$)",eE:!0,k:i,l:a,c:[e.UTM]},{b:"\\."+e.UIR,r:0}]}})),e.registerLanguage("ocaml",(function(e){return{aliases:["ml"],k:{keyword:"and as assert asr begin class constraint do done downto else end exception external for fun function functor if in include inherit! inherit initializer land lazy let lor lsl lsr lxor match method!|10 method mod module mutable new object of open! open or private rec sig struct then to try type val! val virtual when while with parser value",built_in:"array bool bytes char exn|5 float int int32 int64 list lazy_t|5 nativeint|5 string unit in_channel out_channel ref",literal:"true false"},i:/\/\/|>>/,l:"[a-z_]\\w*!?",c:[{cN:"literal",b:"\\[(\\|\\|)?\\]|\\(\\)",r:0},e.C("\\(\\*","\\*\\)",{c:["self"]}),{cN:"symbol",b:"'[A-Za-z_](?!')[\\w']*"},{cN:"type",b:"`[A-Z][\\w']*"},{cN:"type",b:"\\b[A-Z][\\w']*",r:0},{b:"[a-z_]\\w*'[\\w']*",r:0},e.inherit(e.ASM,{cN:"string",r:0}),e.inherit(e.QSM,{i:null}),{cN:"number",b:"\\b(0[xX][a-fA-F0-9_]+[Lln]?|0[oO][0-7_]+[Lln]?|0[bB][01_]+[Lln]?|[0-9][0-9_]*([Lln]|(\\.[0-9_]*)?([eE][-+]?[0-9_]+)?)?)",r:0},{b:/[-=]>/}]}})),e.registerLanguage("openscad",(function(e){var t={cN:"keyword",b:"\\$(f[asn]|t|vp[rtd]|children)"},r={cN:"literal",b:"false|true|PI|undef"},a={cN:"number",b:"\\b\\d+(\\.\\d+)?(e-?\\d+)?",r:0},i=e.inherit(e.QSM,{i:null}),n={cN:"meta",k:{"meta-keyword":"include use"},b:"include|use <",e:">"},o={cN:"params",b:"\\(",e:"\\)",c:["self",a,i,t,r]},s={b:"[*!#%]",r:0},l={cN:"function",bK:"module function",e:"\\=|\\{",c:[o,e.UTM]};return{aliases:["scad"],k:{keyword:"function module include use for intersection_for if else \\%",literal:"false true PI undef",built_in:"circle square polygon text sphere cube cylinder polyhedron translate rotate scale resize mirror multmatrix color offset hull minkowski union difference intersection abs sign sin cos tan acos asin atan atan2 floor round ceil ln log pow sqrt exp rands min max concat lookup str chr search version version_num norm cross parent_module echo import import_dxf dxf_linear_extrude linear_extrude rotate_extrude surface projection render children dxf_cross dxf_dim let assign"},c:[e.CLCM,e.CBCM,a,n,i,t,s,l]}})),e.registerLanguage("oxygene",(function(e){var t="abstract add and array as asc aspect assembly async begin break block by case class concat const copy constructor continue create default delegate desc distinct div do downto dynamic each else empty end ensure enum equals event except exit extension external false final finalize finalizer finally flags for forward from function future global group has if implementation implements implies in index inherited inline interface into invariants is iterator join locked locking loop matching method mod module namespace nested new nil not notify nullable of old on operator or order out override parallel params partial pinned private procedure property protected public queryable raise read readonly record reintroduce remove repeat require result reverse sealed select self sequence set shl shr skip static step soft take then to true try tuple type union unit unsafe until uses using var virtual raises volatile where while with write xor yield await mapped deprecated stdcall cdecl pascal register safecall overload library platform reference packed strict published autoreleasepool selector strong weak unretained",r=e.C("{","}",{r:0}),a=e.C("\\(\\*","\\*\\)",{r:10}),i={cN:"string",b:"'",e:"'",c:[{b:"''"}]},n={cN:"string",b:"(#\\d+)+"},o={cN:"function",bK:"function constructor destructor procedure method",e:"[:;]",k:"function constructor|10 destructor|10 procedure|10 method|10",c:[e.TM,{cN:"params",b:"\\(",e:"\\)",k:t,c:[i,n]},r,a]};return{cI:!0,l:/\.?\w+/,k:t,i:'("|\\$[G-Zg-z]|\\/\\*||->)',c:[r,a,e.CLCM,i,n,e.NM,o,{cN:"class",b:"=\\bclass\\b",e:"end;",k:t,c:[i,n,r,a,e.CLCM,o]}]}})),e.registerLanguage("parser3",(function(e){var t=e.C("{","}",{c:["self"]});return{sL:"xml",r:0,c:[e.C("^#","$"),e.C("\\^rem{","}",{r:10,c:[t]}),{cN:"meta",b:"^@(?:BASE|USE|CLASS|OPTIONS)$",r:10},{cN:"title",b:"@[\\w\\-]+\\[[\\w^;\\-]*\\](?:\\[[\\w^;\\-]*\\])?(?:.*)$"},{cN:"variable",b:"\\$\\{?[\\w\\-\\.\\:]+\\}?"},{cN:"keyword",b:"\\^[\\w\\-\\.\\:]+"},{cN:"number",b:"\\^#[0-9a-fA-F]+"},e.CNM]}})),e.registerLanguage("pf",(function(e){var t={cN:"variable",b:/\$[\w\d#@][\w\d_]*/},r={cN:"variable",b:/<(?!\/)/,e:/>/};return{aliases:["pf.conf"],l:/[a-z0-9_<>-]+/,k:{built_in:"block match pass load anchor|5 antispoof|10 set table",keyword:"in out log quick on rdomain inet inet6 proto from port os to routeallow-opts divert-packet divert-reply divert-to flags group icmp-typeicmp6-type label once probability received-on rtable prio queuetos tag tagged user keep fragment for os dropaf-to|10 binat-to|10 nat-to|10 rdr-to|10 bitmask least-stats random round-robinsource-hash static-portdup-to reply-to route-toparent bandwidth default min max qlimitblock-policy debug fingerprints hostid limit loginterface optimizationreassemble ruleset-optimization basic none profile skip state-defaultsstate-policy timeoutconst counters persistno modulate synproxy state|5 floating if-bound no-sync pflow|10 sloppysource-track global rule max-src-nodes max-src-states max-src-connmax-src-conn-rate overload flushscrub|5 max-mss min-ttl no-df|10 random-id",literal:"all any no-route self urpf-failed egress|5 unknown"},c:[e.HCM,e.NM,e.QSM,t,r]}})),e.registerLanguage("php",(function(e){var t={b:"\\$+[a-zA-Z_-ÿ][a-zA-Z0-9_-ÿ]*"},r={cN:"meta",b:/<\?(php)?|\?>/},a={cN:"string",c:[e.BE,r],v:[{b:'b"',e:'"'},{b:"b'",e:"'"},e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null})]},i={v:[e.BNM,e.CNM]};return{aliases:["php3","php4","php5","php6"],cI:!0,k:"and include_once list abstract global private echo interface as static endswitch array null if endwhile or const for endforeach self var while isset public protected exit foreach throw elseif include __FILE__ empty require_once do xor return parent clone use __CLASS__ __LINE__ else break print eval new catch __METHOD__ case exception default die require __FUNCTION__ enddeclare final try switch continue endfor endif declare unset true false trait goto instanceof insteadof __DIR__ __NAMESPACE__ yield finally",c:[e.HCM,e.C("//","$",{c:[r]}),e.C("/\\*","\\*/",{c:[{cN:"doctag",b:"@[A-Za-z]+"}]}),e.C("__halt_compiler.+?;",!1,{eW:!0,k:"__halt_compiler",l:e.UIR}),{cN:"string",b:/<<<['"]?\w+['"]?$/,e:/^\w+;?$/,c:[e.BE,{cN:"subst",v:[{b:/\$\w+/},{b:/\{\$/,e:/\}/}]}]},r,{cN:"keyword",b:/\$this\b/},t,{b:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{cN:"function",bK:"function",e:/[;{]/,eE:!0,i:"\\$|\\[|%",c:[e.UTM,{cN:"params",b:"\\(",e:"\\)",c:["self",t,e.CBCM,a,i]}]},{cN:"class",bK:"class interface",e:"{",eE:!0,i:/[:\(\$"]/,c:[{bK:"extends implements"},e.UTM]},{bK:"namespace",e:";",i:/[\.']/,c:[e.UTM]},{bK:"use",e:";",c:[e.UTM]},{b:"=>"},a,i]}})),e.registerLanguage("pony",(function(e){var t={keyword:"actor addressof and as be break class compile_error compile_intrinsicconsume continue delegate digestof do else elseif embed end errorfor fun if ifdef in interface is isnt lambda let match new not objector primitive recover repeat return struct then trait try type until use var where while with xor",meta:"iso val tag trn box ref",literal:"this false true"},r={cN:"string",b:'"""',e:'"""',r:10},a={cN:"string",b:'"',e:'"',c:[e.BE]},i={cN:"string",b:"'",e:"'",c:[e.BE],r:0},n={cN:"type",b:"\\b_?[A-Z][\\w]*",r:0},o={b:e.IR+"'",r:0},s={cN:"class",bK:"class actor",e:"$",c:[e.TM,e.CLCM]},l={cN:"function",bK:"new fun",e:"=>",c:[e.TM,{b:/\(/,e:/\)/,c:[n,o,e.CNM,e.CBCM]},{b:/:/,eW:!0,c:[n]},e.CLCM]};return{k:t,c:[s,l,n,r,a,i,o,e.CNM,e.CLCM,e.CBCM]}})),e.registerLanguage("powershell",(function(e){var t={b:"`[\\s\\S]",r:0},r={cN:"variable",v:[{b:/\$[\w\d][\w\d_:]*/}]},a={cN:"literal",b:/\$(null|true|false)\b/},i={cN:"string",v:[{b:/"/,e:/"/},{b:/@"/,e:/^"@/}],c:[t,r,{cN:"variable",b:/\$[A-z]/,e:/[^A-z]/}]},n={cN:"string",v:[{b:/'/,e:/'/},{b:/@'/,e:/^'@/}]},o={cN:"doctag",v:[{b:/\.(synopsis|description|example|inputs|outputs|notes|link|component|role|functionality)/},{b:/\.(parameter|forwardhelptargetname|forwardhelpcategory|remotehelprunspace|externalhelp)\s+\S+/}]},s=e.inherit(e.C(null,null),{v:[{b:/#/,e:/$/},{b:/<#/,e:/#>/}],c:[o]});return{aliases:["ps"],l:/-?[A-z\.\-]+/,cI:!0,k:{keyword:"if else foreach return function do while until elseif begin for trap data dynamicparam end break throw param continue finally in switch exit filter try process catch",built_in:"Add-Computer Add-Content Add-History Add-JobTrigger Add-Member Add-PSSnapin Add-Type Checkpoint-Computer Clear-Content Clear-EventLog Clear-History Clear-Host Clear-Item Clear-ItemProperty Clear-Variable Compare-Object Complete-Transaction Connect-PSSession Connect-WSMan Convert-Path ConvertFrom-Csv ConvertFrom-Json ConvertFrom-SecureString ConvertFrom-StringData ConvertTo-Csv ConvertTo-Html ConvertTo-Json ConvertTo-SecureString ConvertTo-Xml Copy-Item Copy-ItemProperty Debug-Process Disable-ComputerRestore Disable-JobTrigger Disable-PSBreakpoint Disable-PSRemoting Disable-PSSessionConfiguration Disable-WSManCredSSP Disconnect-PSSession Disconnect-WSMan Disable-ScheduledJob Enable-ComputerRestore Enable-JobTrigger Enable-PSBreakpoint Enable-PSRemoting Enable-PSSessionConfiguration Enable-ScheduledJob Enable-WSManCredSSP Enter-PSSession Exit-PSSession Export-Alias Export-Clixml Export-Console Export-Counter Export-Csv Export-FormatData Export-ModuleMember Export-PSSession ForEach-Object Format-Custom Format-List Format-Table Format-Wide Get-Acl Get-Alias Get-AuthenticodeSignature Get-ChildItem Get-Command Get-ComputerRestorePoint Get-Content Get-ControlPanelItem Get-Counter Get-Credential Get-Culture Get-Date Get-Event Get-EventLog Get-EventSubscriber Get-ExecutionPolicy Get-FormatData Get-Host Get-HotFix Get-Help Get-History Get-IseSnippet Get-Item Get-ItemProperty Get-Job Get-JobTrigger Get-Location Get-Member Get-Module Get-PfxCertificate Get-Process Get-PSBreakpoint Get-PSCallStack Get-PSDrive Get-PSProvider Get-PSSession Get-PSSessionConfiguration Get-PSSnapin Get-Random Get-ScheduledJob Get-ScheduledJobOption Get-Service Get-TraceSource Get-Transaction Get-TypeData Get-UICulture Get-Unique Get-Variable Get-Verb Get-WinEvent Get-WmiObject Get-WSManCredSSP Get-WSManInstance Group-Object Import-Alias Import-Clixml Import-Counter Import-Csv Import-IseSnippet Import-LocalizedData Import-PSSession Import-Module Invoke-AsWorkflow Invoke-Command Invoke-Expression Invoke-History Invoke-Item Invoke-RestMethod Invoke-WebRequest Invoke-WmiMethod Invoke-WSManAction Join-Path Limit-EventLog Measure-Command Measure-Object Move-Item Move-ItemProperty New-Alias New-Event New-EventLog New-IseSnippet New-Item New-ItemProperty New-JobTrigger New-Object New-Module New-ModuleManifest New-PSDrive New-PSSession New-PSSessionConfigurationFile New-PSSessionOption New-PSTransportOption New-PSWorkflowExecutionOption New-PSWorkflowSession New-ScheduledJobOption New-Service New-TimeSpan New-Variable New-WebServiceProxy New-WinEvent New-WSManInstance New-WSManSessionOption Out-Default Out-File Out-GridView Out-Host Out-Null Out-Printer Out-String Pop-Location Push-Location Read-Host Receive-Job Register-EngineEvent Register-ObjectEvent Register-PSSessionConfiguration Register-ScheduledJob Register-WmiEvent Remove-Computer Remove-Event Remove-EventLog Remove-Item Remove-ItemProperty Remove-Job Remove-JobTrigger Remove-Module Remove-PSBreakpoint Remove-PSDrive Remove-PSSession Remove-PSSnapin Remove-TypeData Remove-Variable Remove-WmiObject Remove-WSManInstance Rename-Computer Rename-Item Rename-ItemProperty Reset-ComputerMachinePassword Resolve-Path Restart-Computer Restart-Service Restore-Computer Resume-Job Resume-Service Save-Help Select-Object Select-String Select-Xml Send-MailMessage Set-Acl Set-Alias Set-AuthenticodeSignature Set-Content Set-Date Set-ExecutionPolicy Set-Item Set-ItemProperty Set-JobTrigger Set-Location Set-PSBreakpoint Set-PSDebug Set-PSSessionConfiguration Set-ScheduledJob Set-ScheduledJobOption Set-Service Set-StrictMode Set-TraceSource Set-Variable Set-WmiInstance Set-WSManInstance Set-WSManQuickConfig Show-Command Show-ControlPanelItem Show-EventLog Sort-Object Split-Path Start-Job Start-Process Start-Service Start-Sleep Start-Transaction Start-Transcript Stop-Computer Stop-Job Stop-Process Stop-Service Stop-Transcript Suspend-Job Suspend-Service Tee-Object Test-ComputerSecureChannel Test-Connection Test-ModuleManifest Test-Path Test-PSSessionConfigurationFile Trace-Command Unblock-File Undo-Transaction Unregister-Event Unregister-PSSessionConfiguration Unregister-ScheduledJob Update-FormatData Update-Help Update-List Update-TypeData Use-Transaction Wait-Event Wait-Job Wait-Process Where-Object Write-Debug Write-Error Write-EventLog Write-Host Write-Output Write-Progress Write-Verbose Write-Warning Add-MDTPersistentDrive Disable-MDTMonitorService Enable-MDTMonitorService Get-MDTDeploymentShareStatistics Get-MDTMonitorData Get-MDTOperatingSystemCatalog Get-MDTPersistentDrive Import-MDTApplication Import-MDTDriver Import-MDTOperatingSystem Import-MDTPackage Import-MDTTaskSequence New-MDTDatabase Remove-MDTMonitorData Remove-MDTPersistentDrive Restore-MDTPersistentDrive Set-MDTMonitorData Test-MDTDeploymentShare Test-MDTMonitorData Update-MDTDatabaseSchema Update-MDTDeploymentShare Update-MDTLinkedDS Update-MDTMedia Update-MDTMedia Add-VamtProductKey Export-VamtData Find-VamtManagedMachine Get-VamtConfirmationId Get-VamtProduct Get-VamtProductKey Import-VamtData Initialize-VamtData Install-VamtConfirmationId Install-VamtProductActivation Install-VamtProductKey Update-VamtProduct",nomarkup:"-ne -eq -lt -gt -ge -le -not -like -notlike -match -notmatch -contains -notcontains -in -notin -replace"},c:[t,e.NM,i,n,a,r,s]}})),e.registerLanguage("processing",(function(e){return{k:{keyword:"BufferedReader PVector PFont PImage PGraphics HashMap boolean byte char color double float int long String Array FloatDict FloatList IntDict IntList JSONArray JSONObject Object StringDict StringList Table TableRow XML false synchronized int abstract float private char boolean static null if const for true while long throw strictfp finally protected import native final return void enum else break transient new catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private",literal:"P2D P3D HALF_PI PI QUARTER_PI TAU TWO_PI",title:"setup draw",built_in:"displayHeight displayWidth mouseY mouseX mousePressed pmouseX pmouseY key keyCode pixels focused frameCount frameRate height width size createGraphics beginDraw createShape loadShape PShape arc ellipse line point quad rect triangle bezier bezierDetail bezierPoint bezierTangent curve curveDetail curvePoint curveTangent curveTightness shape shapeMode beginContour beginShape bezierVertex curveVertex endContour endShape quadraticVertex vertex ellipseMode noSmooth rectMode smooth strokeCap strokeJoin strokeWeight mouseClicked mouseDragged mouseMoved mousePressed mouseReleased mouseWheel keyPressed keyPressedkeyReleased keyTyped print println save saveFrame day hour millis minute month second year background clear colorMode fill noFill noStroke stroke alpha blue brightness color green hue lerpColor red saturation modelX modelY modelZ screenX screenY screenZ ambient emissive shininess specular add createImage beginCamera camera endCamera frustum ortho perspective printCamera printProjection cursor frameRate noCursor exit loop noLoop popStyle pushStyle redraw binary boolean byte char float hex int str unbinary unhex join match matchAll nf nfc nfp nfs split splitTokens trim append arrayCopy concat expand reverse shorten sort splice subset box sphere sphereDetail createInput createReader loadBytes loadJSONArray loadJSONObject loadStrings loadTable loadXML open parseXML saveTable selectFolder selectInput beginRaw beginRecord createOutput createWriter endRaw endRecord PrintWritersaveBytes saveJSONArray saveJSONObject saveStream saveStrings saveXML selectOutput popMatrix printMatrix pushMatrix resetMatrix rotate rotateX rotateY rotateZ scale shearX shearY translate ambientLight directionalLight lightFalloff lights lightSpecular noLights normal pointLight spotLight image imageMode loadImage noTint requestImage tint texture textureMode textureWrap blend copy filter get loadPixels set updatePixels blendMode loadShader PShaderresetShader shader createFont loadFont text textFont textAlign textLeading textMode textSize textWidth textAscent textDescent abs ceil constrain dist exp floor lerp log mag map max min norm pow round sq sqrt acos asin atan atan2 cos degrees radians sin tan noise noiseDetail noiseSeed random randomGaussian randomSeed"},c:[e.CLCM,e.CBCM,e.ASM,e.QSM,e.CNM]}})),e.registerLanguage("profile",(function(e){return{c:[e.CNM,{b:"[a-zA-Z_][\\da-zA-Z_]+\\.[\\da-zA-Z_]{1,3}",e:":",eE:!0},{b:"(ncalls|tottime|cumtime)",e:"$",k:"ncalls tottime|10 cumtime|10 filename",r:10},{b:"function calls",e:"$",c:[e.CNM],r:10},e.ASM,e.QSM,{cN:"string",b:"\\(",e:"\\)$",eB:!0,eE:!0,r:0}]}})),e.registerLanguage("prolog",(function(e){var t={b:/[a-z][A-Za-z0-9_]*/,r:0},r={cN:"symbol",v:[{b:/[A-Z][a-zA-Z0-9_]*/},{b:/_[A-Za-z0-9_]*/}],r:0},a={b:/\(/,e:/\)/,r:0},i={b:/\[/,e:/\]/},n={cN:"comment",b:/%/,e:/$/,c:[e.PWM]},o={cN:"string",b:/`/,e:/`/,c:[e.BE]},s={cN:"string",b:/0\'(\\\'|.)/},l={cN:"string",b:/0\'\\s/},c={b:/:-/},d=[t,r,a,c,i,n,e.CBCM,e.QSM,e.ASM,o,s,l,e.CNM];return a.c=d,i.c=d,{c:d.concat([{b:/\.$/}])}})),e.registerLanguage("protobuf",(function(e){return{k:{keyword:"package import option optional required repeated group",built_in:"double float int32 int64 uint32 uint64 sint32 sint64 fixed32 fixed64 sfixed32 sfixed64 bool string bytes",literal:"true false"},c:[e.QSM,e.NM,e.CLCM,{cN:"class",bK:"message enum service",e:/\{/,i:/\n/,c:[e.inherit(e.TM,{starts:{eW:!0,eE:!0}})]},{cN:"function",bK:"rpc",e:/;/,eE:!0,k:"rpc returns"},{b:/^\s*[A-Z_]+/,e:/\s*=/,eE:!0}]}})),e.registerLanguage("puppet",(function(e){var t={keyword:"and case default else elsif false if in import enherits node or true undef unless main settings $string ",literal:"alias audit before loglevel noop require subscribe tag owner ensure group mode name|0 changes context force incl lens load_path onlyif provider returns root show_diff type_check en_address ip_address realname command environment hour monute month monthday special target weekday creates cwd ogoutput refresh refreshonly tries try_sleep umask backup checksum content ctime force ignore links mtime purge recurse recurselimit replace selinux_ignore_defaults selrange selrole seltype seluser source souirce_permissions sourceselect validate_cmd validate_replacement allowdupe attribute_membership auth_membership forcelocal gid ia_load_module members system host_aliases ip allowed_trunk_vlans description device_url duplex encapsulation etherchannel native_vlan speed principals allow_root auth_class auth_type authenticate_user k_of_n mechanisms rule session_owner shared options device fstype enable hasrestart directory present absent link atboot blockdevice device dump pass remounts poller_tag use message withpath adminfile allow_virtual allowcdrom category configfiles flavor install_options instance package_settings platform responsefile status uninstall_options vendor unless_system_user unless_uid binary control flags hasstatus manifest pattern restart running start stop allowdupe auths expiry gid groups home iterations key_membership keys managehome membership password password_max_age password_min_age profile_membership profiles project purge_ssh_keys role_membership roles salt shell uid baseurl cost descr enabled enablegroups exclude failovermethod gpgcheck gpgkey http_caching include includepkgs keepalive metadata_expire metalink mirrorlist priority protect proxy proxy_password proxy_username repo_gpgcheck s3_enabled skip_if_unavailable sslcacert sslclientcert sslclientkey sslverify mounted",built_in:"architecture augeasversion blockdevices boardmanufacturer boardproductname boardserialnumber cfkey dhcp_servers domain ec2_ ec2_userdata facterversion filesystems ldom fqdn gid hardwareisa hardwaremodel hostname id|0 interfaces ipaddress ipaddress_ ipaddress6 ipaddress6_ iphostnumber is_virtual kernel kernelmajversion kernelrelease kernelversion kernelrelease kernelversion lsbdistcodename lsbdistdescription lsbdistid lsbdistrelease lsbmajdistrelease lsbminordistrelease lsbrelease macaddress macaddress_ macosx_buildversion macosx_productname macosx_productversion macosx_productverson_major macosx_productversion_minor manufacturer memoryfree memorysize netmask metmask_ network_ operatingsystem operatingsystemmajrelease operatingsystemrelease osfamily partitions path physicalprocessorcount processor processorcount productname ps puppetversion rubysitedir rubyversion selinux selinux_config_mode selinux_config_policy selinux_current_mode selinux_current_mode selinux_enforced selinux_policyversion serialnumber sp_ sshdsakey sshecdsakey sshrsakey swapencrypted swapfree swapsize timezone type uniqueid uptime uptime_days uptime_hours uptime_seconds uuid virtual vlans xendomains zfs_version zonenae zones zpool_version"},r=e.C("#","$"),a="([A-Za-z_]|::)(\\w|::)*",i=e.inherit(e.TM,{b:a}),n={cN:"variable",b:"\\$"+a},o={cN:"string",c:[e.BE,n],v:[{b:/'/,e:/'/},{b:/"/,e:/"/}]};return{aliases:["pp"],c:[r,n,o,{bK:"class",e:"\\{|;",i:/=/,c:[i,r]},{bK:"define",e:/\{/,c:[{cN:"section",b:e.IR,endsParent:!0}]},{b:e.IR+"\\s+\\{",rB:!0,e:/\S/,c:[{cN:"keyword",b:e.IR},{b:/\{/,e:/\}/,k:t,r:0,c:[o,r,{b:"[a-zA-Z_]+\\s*=>",rB:!0,e:"=>",c:[{cN:"attr",b:e.IR}]},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},n]}],r:0}]}})),e.registerLanguage("purebasic",(function(e){var t={cN:"string",b:'(~)?"',e:'"',i:"\\n"},r={cN:"symbol",b:"#[a-zA-Z_]\\w*\\$?"};return{aliases:["pb","pbi"],k:"And As Break CallDebugger Case CompilerCase CompilerDefault CompilerElse CompilerEndIf CompilerEndSelect CompilerError CompilerIf CompilerSelect Continue Data DataSection EndDataSection Debug DebugLevel Default Define Dim DisableASM DisableDebugger DisableExplicit Else ElseIf EnableASM EnableDebugger EnableExplicit End EndEnumeration EndIf EndImport EndInterface EndMacro EndProcedure EndSelect EndStructure EndStructureUnion EndWith Enumeration Extends FakeReturn For Next ForEach ForEver Global Gosub Goto If Import ImportC IncludeBinary IncludeFile IncludePath Interface Macro NewList Not Or ProcedureReturn Protected Prototype PrototypeC Read ReDim Repeat Until Restore Return Select Shared Static Step Structure StructureUnion Swap To Wend While With XIncludeFile XOr Procedure ProcedureC ProcedureCDLL ProcedureDLL Declare DeclareC DeclareCDLL DeclareDLL",c:[e.C(";","$",{r:0}),{cN:"function",b:"\\b(Procedure|Declare)(C|CDLL|DLL)?\\b",e:"\\(",eE:!0,rB:!0,c:[{cN:"keyword",b:"(Procedure|Declare)(C|CDLL|DLL)?",eE:!0},{cN:"type",b:"\\.\\w*"},e.UTM]},t,r]}})),e.registerLanguage("python",(function(e){var t={keyword:"and elif is global as in if from raise for except finally print import pass return exec else break not with class assert yield try while continue del or def lambda async await nonlocal|10 None True False",built_in:"Ellipsis NotImplemented"},r={cN:"meta",b:/^(>>>|\.\.\.) /},a={cN:"subst",b:/\{/,e:/\}/,k:t,i:/#/},i={cN:"string",c:[e.BE],v:[{b:/(u|b)?r?'''/,e:/'''/,c:[r],r:10},{b:/(u|b)?r?"""/,e:/"""/,c:[r],r:10},{b:/(fr|rf|f)'''/,e:/'''/,c:[r,a]},{b:/(fr|rf|f)"""/,e:/"""/,c:[r,a]},{b:/(u|r|ur)'/,e:/'/,r:10},{b:/(u|r|ur)"/,e:/"/,r:10},{b:/(b|br)'/,e:/'/},{b:/(b|br)"/,e:/"/},{b:/(fr|rf|f)'/,e:/'/,c:[a]},{b:/(fr|rf|f)"/,e:/"/,c:[a]},e.ASM,e.QSM]},n={cN:"number",r:0,v:[{b:e.BNR+"[lLjJ]?"},{b:"\\b(0o[0-7]+)[lLjJ]?"},{b:e.CNR+"[lLjJ]?"}]},o={cN:"params",b:/\(/,e:/\)/,c:["self",r,n,i]};return a.c=[i,n,r],{aliases:["py","gyp"],k:t,i:/(<\/|->|\?)|=>/,c:[r,n,i,e.HCM,{v:[{cN:"function",bK:"def"},{cN:"class",bK:"class"}],e:/:/,i:/[${=;\n,]/,c:[e.UTM,o,{b:/->/,eW:!0,k:"None"}]},{cN:"meta",b:/^[\t ]*@/,e:/$/},{b:/\b(print|exec)\(/}]}})),e.registerLanguage("q",(function(e){var t={keyword:"do while select delete by update from",literal:"0b 1b",built_in:"neg not null string reciprocal floor ceiling signum mod xbar xlog and or each scan over prior mmu lsq inv md5 ltime gtime count first var dev med cov cor all any rand sums prds mins maxs fills deltas ratios avgs differ prev next rank reverse iasc idesc asc desc msum mcount mavg mdev xrank mmin mmax xprev rotate distinct group where flip type key til get value attr cut set upsert raze union inter except cross sv vs sublist enlist read0 read1 hopen hclose hdel hsym hcount peach system ltrim rtrim trim lower upper ssr view tables views cols xcols keys xkey xcol xasc xdesc fkeys meta lj aj aj0 ij pj asof uj ww wj wj1 fby xgroup ungroup ej save load rsave rload show csv parse eval min max avg wavg wsum sin cos tan sum",type:"`float `double int `timestamp `timespan `datetime `time `boolean `symbol `char `byte `short `long `real `month `date `minute `second `guid"};return{aliases:["k","kdb"],k:t,l:/(`?)[A-Za-z0-9_]+\b/,c:[e.CLCM,e.QSM,e.CNM]}})),e.registerLanguage("qml",(function(e){var t={keyword:"in of on if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await import",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Behavior bool color coordinate date double enumeration font geocircle georectangle geoshape int list matrix4x4 parent point quaternion real rect size string url variant vector2d vector3d vector4dPromise"},r="[a-zA-Z_][a-zA-Z0-9\\._]*",a={cN:"keyword",b:"\\bproperty\\b",starts:{cN:"string",e:"(:|=|;|,|//|/\\*|$)",rE:!0}},i={cN:"keyword",b:"\\bsignal\\b",starts:{cN:"string",e:"(\\(|:|=|;|,|//|/\\*|$)",rE:!0}},n={cN:"attribute",b:"\\bid\\s*:",starts:{cN:"string",e:r,rE:!1}},o={b:r+"\\s*:",rB:!0,c:[{cN:"attribute",b:r,e:"\\s*:",eE:!0,r:0}],r:0},s={b:r+"\\s*{",e:"{",rB:!0,r:0,c:[e.inherit(e.TM,{b:r})]};return{aliases:["qt"],cI:!1,k:t,c:[{cN:"meta",b:/^\s*['"]use (strict|asm)['"]/},e.ASM,e.QSM,{cN:"string",b:"`",e:"`",c:[e.BE,{cN:"subst",b:"\\$\\{",e:"\\}"}]},e.CLCM,e.CBCM,{cN:"number",v:[{b:"\\b(0[bB][01]+)"},{b:"\\b(0[oO][0-7]+)"},{b:e.CNR}],r:0},{b:"("+e.RSR+"|\\b(case|return|throw)\\b)\\s*",k:"return throw case",c:[e.CLCM,e.CBCM,e.RM,{b:/\s*[);\]]/,r:0,sL:"xml"}],r:0},i,a,{cN:"function",bK:"function",e:/\{/,eE:!0,c:[e.inherit(e.TM,{b:/[A-Za-z$_][0-9A-Za-z$_]*/}),{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,c:[e.CLCM,e.CBCM]}],i:/\[|%/},{b:"\\."+e.IR,r:0},n,o,s],i:/#/}})),e.registerLanguage("r",(function(e){var t="([a-zA-Z]|\\.[a-zA-Z.])[a-zA-Z0-9._]*";return{c:[e.HCM,{b:t,l:t,k:{keyword:"function if in break next repeat else for return switch while try tryCatch stop warning require library attach detach source setMethod setGeneric setGroupGeneric setClass ...",literal:"NULL NA TRUE FALSE T F Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10"},r:0},{cN:"number",b:"0[xX][0-9a-fA-F]+[Li]?\\b",r:0},{cN:"number",b:"\\d+(?:[eE][+\\-]?\\d*)?L\\b",r:0},{cN:"number",b:"\\d+\\.(?!\\d)(?:i\\b)?",r:0},{cN:"number",b:"\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",r:0},{cN:"number",b:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",r:0},{b:"`",e:"`",r:0},{cN:"string",c:[e.BE],v:[{b:'"',e:'"'},{b:"'",e:"'"}]}]}})),e.registerLanguage("rib",(function(e){return{k:"ArchiveRecord AreaLightSource Atmosphere Attribute AttributeBegin AttributeEnd Basis Begin Blobby Bound Clipping ClippingPlane Color ColorSamples ConcatTransform Cone CoordinateSystem CoordSysTransform CropWindow Curves Cylinder DepthOfField Detail DetailRange Disk Displacement Display End ErrorHandler Exposure Exterior Format FrameAspectRatio FrameBegin FrameEnd GeneralPolygon GeometricApproximation Geometry Hider Hyperboloid Identity Illuminate Imager Interior LightSource MakeCubeFaceEnvironment MakeLatLongEnvironment MakeShadow MakeTexture Matte MotionBegin MotionEnd NuPatch ObjectBegin ObjectEnd ObjectInstance Opacity Option Orientation Paraboloid Patch PatchMesh Perspective PixelFilter PixelSamples PixelVariance Points PointsGeneralPolygons PointsPolygons Polygon Procedural Projection Quantize ReadArchive RelativeDetail ReverseOrientation Rotate Scale ScreenWindow ShadingInterpolation ShadingRate Shutter Sides Skew SolidBegin SolidEnd Sphere SubdivisionMesh Surface TextureCoordinates Torus Transform TransformBegin TransformEnd TransformPoints Translate TrimCurve WorldBegin WorldEnd",i:"\]$/},{b:/<\//,e:/>/},{b:/^facet /,e:/\}/},{b:"^1\\.\\.(\\d+)$",e:/$/}],i:/./},e.C("^#","$"),s,l,o,{b:/[\w-]+\=([^\s\{\}\[\]\(\)]+)/,r:0,rB:!0,c:[{cN:"attribute",b:/[^=]+/},{b:/=/,eW:!0,r:0,c:[s,l,o,{cN:"literal",b:"\\b("+i.split(" ").join("|")+")\\b"},{b:/("[^"]*"|[^\s\{\}\[\]]+)/}]}]},{cN:"number",b:/\*[0-9a-fA-F]+/},{b:"\\b("+a.split(" ").join("|")+")([\\s[(]|])",rB:!0,c:[{cN:"builtin-name",b:/\w+/}]},{cN:"built_in",v:[{b:"(\\.\\./|/|\\s)(("+n.split(" ").join("|")+");?\\s)+",r:10},{b:/\.\./}]}]}})),e.registerLanguage("rsl",(function(e){return{k:{keyword:"float color point normal vector matrix while for if do return else break extern continue",built_in:"abs acos ambient area asin atan atmosphere attribute calculatenormal ceil cellnoise clamp comp concat cos degrees depth Deriv diffuse distance Du Dv environment exp faceforward filterstep floor format fresnel incident length lightsource log match max min mod noise normalize ntransform opposite option phong pnoise pow printf ptlined radians random reflect refract renderinfo round setcomp setxcomp setycomp setzcomp shadow sign sin smoothstep specular specularbrdf spline sqrt step tan texture textureinfo trace transform vtransform xcomp ycomp zcomp"},i:""}]}})),e.registerLanguage("scala",(function(e){var t={cN:"meta",b:"@[A-Za-z]+"},r={cN:"subst",v:[{b:"\\$[A-Za-z0-9_]+"},{b:"\\${",e:"}"}]},a={cN:"string",v:[{b:'"',e:'"',i:"\\n",c:[e.BE]},{b:'"""',e:'"""',r:10},{b:'[a-z]+"',e:'"',i:"\\n",c:[e.BE,r]},{cN:"string",b:'[a-z]+"""',e:'"""',c:[r],r:10}]},i={cN:"symbol",b:"'\\w[\\w\\d_]*(?!')"},n={cN:"type",b:"\\b[A-Z][A-Za-z0-9_]*",r:0},o={cN:"title",b:/[^0-9\n\t "'(),.`{}\[\]:;][^\n\t "'(),.`{}\[\]:;]+|[^0-9\n\t "'(),.`{}\[\]:;=]/,r:0},s={cN:"class",bK:"class object trait type",e:/[:={\[\n;]/,eE:!0,c:[{bK:"extends with",r:10},{b:/\[/,e:/\]/,eB:!0,eE:!0,r:0,c:[n]},{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,r:0,c:[n]},o]},l={cN:"function",bK:"def",e:/[:={\[(\n;]/,eE:!0,c:[o]};return{k:{literal:"true false null",keyword:"type yield lazy override def with val var sealed abstract private trait object if forSome for while throw finally protected extends import final return else break new catch super class case package default try this match continue throws implicit"},c:[e.CLCM,e.CBCM,a,i,n,l,s,e.CNM,t]}})),e.registerLanguage("scheme",(function(e){var t="[^\\(\\)\\[\\]\\{\\}\",'`;#|\\\\\\s]+",r="(\\-|\\+)?\\d+([./]\\d+)?",a=r+"[+\\-]"+r+"i",i={"builtin-name":"case-lambda call/cc class define-class exit-handler field import inherit init-field interface let*-values let-values let/ec mixin opt-lambda override protect provide public rename require require-for-syntax syntax syntax-case syntax-error unit/sig unless when with-syntax and begin call-with-current-continuation call-with-input-file call-with-output-file case cond define define-syntax delay do dynamic-wind else for-each if lambda let let* let-syntax letrec letrec-syntax map or syntax-rules ' * + , ,@ - ... / ; < <= = => > >= ` abs acos angle append apply asin assoc assq assv atan boolean? caar cadr call-with-input-file call-with-output-file call-with-values car cdddar cddddr cdr ceiling char->integer char-alphabetic? char-ci<=? char-ci=? char-ci>? char-downcase char-lower-case? char-numeric? char-ready? char-upcase char-upper-case? char-whitespace? char<=? char=? char>? char? close-input-port close-output-port complex? cons cos current-input-port current-output-port denominator display eof-object? eq? equal? eqv? eval even? exact->inexact exact? exp expt floor force gcd imag-part inexact->exact inexact? input-port? integer->char integer? interaction-environment lcm length list list->string list->vector list-ref list-tail list? load log magnitude make-polar make-rectangular make-string make-vector max member memq memv min modulo negative? newline not null-environment null? number->string number? numerator odd? open-input-file open-output-file output-port? pair? peek-char port? positive? procedure? quasiquote quote quotient rational? rationalize read read-char real-part real? remainder reverse round scheme-report-environment set! set-car! set-cdr! sin sqrt string string->list string->number string->symbol string-append string-ci<=? string-ci=? string-ci>? string-copy string-fill! string-length string-ref string-set! string<=? string=? string>? string? substring symbol->string symbol? tan transcript-off transcript-on truncate values vector vector->list vector-fill! vector-length vector-ref vector-set! with-input-from-file with-output-to-file write write-char zero?"},n={cN:"meta",b:"^#!",e:"$"},o={cN:"literal",b:"(#t|#f|#\\\\"+t+"|#\\\\.)"},s={cN:"number",v:[{b:r,r:0},{b:a,r:0},{b:"#b[0-1]+(/[0-1]+)?"},{b:"#o[0-7]+(/[0-7]+)?"},{b:"#x[0-9a-f]+(/[0-9a-f]+)?"}]},l=e.QSM,c=[e.C(";","$",{r:0}),e.C("#\\|","\\|#")],d={b:t,r:0},p={cN:"symbol",b:"'"+t},m={eW:!0,r:0},u={v:[{b:/'/},{b:"`"}],c:[{b:"\\(",e:"\\)",c:["self",o,l,s,d,p]}]},b={cN:"name",b:t,l:t,k:i},g={b:/lambda/,eW:!0,rB:!0,c:[b,{b:/\(/,e:/\)/,endsParent:!0,c:[d]}]},f={v:[{b:"\\(",e:"\\)"},{b:"\\[",e:"\\]"}],c:[g,b,m]};return m.c=[o,s,l,d,p,u,f].concat(c),{i:/\S/,c:[n,s,l,p,u,f].concat(c)}})),e.registerLanguage("scilab",(function(e){var t=[e.CNM,{cN:"string",b:"'|\"",e:"'|\"",c:[e.BE,{b:"''"}]}];return{aliases:["sci"],l:/%?\w+/,k:{keyword:"abort break case clear catch continue do elseif else endfunction end for function global if pause return resume select try then while",literal:"%f %F %t %T %pi %eps %inf %nan %e %i %z %s",built_in:"abs and acos asin atan ceil cd chdir clearglobal cosh cos cumprod deff disp error exec execstr exists exp eye gettext floor fprintf fread fsolve imag isdef isempty isinfisnan isvector lasterror length load linspace list listfiles log10 log2 log max min msprintf mclose mopen ones or pathconvert poly printf prod pwd rand real round sinh sin size gsort sprintf sqrt strcat strcmps tring sum system tanh tan type typename warning zeros matrix"},i:'("|#|/\\*|\\s+/\\w+)',c:[{cN:"function",bK:"function",e:"$",c:[e.UTM,{cN:"params",b:"\\(",e:"\\)"}]},{b:"[a-zA-Z_][a-zA-Z_0-9]*('+[\\.']*|[\\.']+)",e:"",r:0},{b:"\\[",e:"\\]'*[\\.']*",r:0,c:t},e.C("//","$")].concat(t)}})),e.registerLanguage("scss",(function(e){var t="[a-zA-Z-][a-zA-Z0-9_-]*",r={cN:"variable",b:"(\\$"+t+")\\b"},a={cN:"number",b:"#[0-9A-Fa-f]+"};({cN:"attribute",b:"[A-Z\\_\\.\\-]+",e:":",eE:!0,i:"[^\\s]",starts:{eW:!0,eE:!0,c:[a,e.CSSNM,e.QSM,e.ASM,e.CBCM,{cN:"meta",b:"!important"}]}});return{cI:!0,i:"[=/|']",c:[e.CLCM,e.CBCM,{cN:"selector-id",b:"\\#[A-Za-z0-9_-]+",r:0},{cN:"selector-class",b:"\\.[A-Za-z0-9_-]+",r:0},{cN:"selector-attr",b:"\\[",e:"\\]",i:"$"},{cN:"selector-tag",b:"\\b(a|abbr|acronym|address|area|article|aside|audio|b|base|big|blockquote|body|br|button|canvas|caption|cite|code|col|colgroup|command|datalist|dd|del|details|dfn|div|dl|dt|em|embed|fieldset|figcaption|figure|footer|form|frame|frameset|(h[1-6])|head|header|hgroup|hr|html|i|iframe|img|input|ins|kbd|keygen|label|legend|li|link|map|mark|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|pre|progress|q|rp|rt|ruby|samp|script|section|select|small|span|strike|strong|style|sub|sup|table|tbody|td|textarea|tfoot|th|thead|time|title|tr|tt|ul|var|video)\\b",r:0},{b:":(visited|valid|root|right|required|read-write|read-only|out-range|optional|only-of-type|only-child|nth-of-type|nth-last-of-type|nth-last-child|nth-child|not|link|left|last-of-type|last-child|lang|invalid|indeterminate|in-range|hover|focus|first-of-type|first-line|first-letter|first-child|first|enabled|empty|disabled|default|checked|before|after|active)"},{b:"::(after|before|choices|first-letter|first-line|repeat-index|repeat-item|selection|value)"},r,{cN:"attribute",b:"\\b(z-index|word-wrap|word-spacing|word-break|width|widows|white-space|visibility|vertical-align|unicode-bidi|transition-timing-function|transition-property|transition-duration|transition-delay|transition|transform-style|transform-origin|transform|top|text-underline-position|text-transform|text-shadow|text-rendering|text-overflow|text-indent|text-decoration-style|text-decoration-line|text-decoration-color|text-decoration|text-align-last|text-align|tab-size|table-layout|right|resize|quotes|position|pointer-events|perspective-origin|perspective|page-break-inside|page-break-before|page-break-after|padding-top|padding-right|padding-left|padding-bottom|padding|overflow-y|overflow-x|overflow-wrap|overflow|outline-width|outline-style|outline-offset|outline-color|outline|orphans|order|opacity|object-position|object-fit|normal|none|nav-up|nav-right|nav-left|nav-index|nav-down|min-width|min-height|max-width|max-height|mask|marks|margin-top|margin-right|margin-left|margin-bottom|margin|list-style-type|list-style-position|list-style-image|list-style|line-height|letter-spacing|left|justify-content|initial|inherit|ime-mode|image-orientation|image-resolution|image-rendering|icon|hyphens|height|font-weight|font-variant-ligatures|font-variant|font-style|font-stretch|font-size-adjust|font-size|font-language-override|font-kerning|font-feature-settings|font-family|font|float|flex-wrap|flex-shrink|flex-grow|flex-flow|flex-direction|flex-basis|flex|filter|empty-cells|display|direction|cursor|counter-reset|counter-increment|content|column-width|column-span|column-rule-width|column-rule-style|column-rule-color|column-rule|column-gap|column-fill|column-count|columns|color|clip-path|clip|clear|caption-side|break-inside|break-before|break-after|box-sizing|box-shadow|box-decoration-break|bottom|border-width|border-top-width|border-top-style|border-top-right-radius|border-top-left-radius|border-top-color|border-top|border-style|border-spacing|border-right-width|border-right-style|border-right-color|border-right|border-radius|border-left-width|border-left-style|border-left-color|border-left|border-image-width|border-image-source|border-image-slice|border-image-repeat|border-image-outset|border-image|border-color|border-collapse|border-bottom-width|border-bottom-style|border-bottom-right-radius|border-bottom-left-radius|border-bottom-color|border-bottom|border|background-size|background-repeat|background-position|background-origin|background-image|background-color|background-clip|background-attachment|background-blend-mode|background|backface-visibility|auto|animation-timing-function|animation-play-state|animation-name|animation-iteration-count|animation-fill-mode|animation-duration|animation-direction|animation-delay|animation|align-self|align-items|align-content)\\b",i:"[^\\s]"},{b:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b"},{b:":",e:";",c:[r,a,e.CSSNM,e.QSM,e.ASM,{cN:"meta",b:"!important"}]},{b:"@",e:"[{;]",k:"mixin include extend for if else each while charset import debug media page content font-face namespace warn",c:[r,e.QSM,e.ASM,a,e.CSSNM,{b:"\\s[A-Za-z0-9_.-]+",r:0}]}]}})),e.registerLanguage("shell",(function(e){return{aliases:["console"],c:[{cN:"meta",b:"^\\s{0,3}[\\w\\d\\[\\]()@-]*[>%$#]",starts:{e:"$",sL:"bash"}}]}})),e.registerLanguage("smali",(function(e){var t=["add","and","cmp","cmpg","cmpl","const","div","double","float","goto","if","int","long","move","mul","neg","new","nop","not","or","rem","return","shl","shr","sput","sub","throw","ushr","xor"],r=["aget","aput","array","check","execute","fill","filled","goto/16","goto/32","iget","instance","invoke","iput","monitor","packed","sget","sparse"],a=["transient","constructor","abstract","final","synthetic","public","private","protected","static","bridge","system"];return{aliases:["smali"],c:[{cN:"string",b:'"',e:'"',r:0},e.C("#","$",{r:0}),{cN:"keyword",v:[{b:"\\s*\\.end\\s[a-zA-Z0-9]*"},{b:"^[ ]*\\.[a-zA-Z]*",r:0},{b:"\\s:[a-zA-Z_0-9]*",r:0},{b:"\\s("+a.join("|")+")"}]},{cN:"built_in",v:[{b:"\\s("+t.join("|")+")\\s"},{b:"\\s("+t.join("|")+")((\\-|/)[a-zA-Z0-9]+)+\\s",r:10},{b:"\\s("+r.join("|")+")((\\-|/)[a-zA-Z0-9]+)*\\s",r:10}]},{cN:"class",b:"L[^(;:\n]*;",r:0},{b:"[vp][0-9]+"}]}})),e.registerLanguage("smalltalk",(function(e){var t="[a-z][a-zA-Z0-9_]*",r={cN:"string",b:"\\$.{1}"},a={cN:"symbol",b:"#"+e.UIR};return{aliases:["st"],k:"self super nil true false thisContext",c:[e.C('"','"'),e.ASM,{cN:"type",b:"\\b[A-Z][A-Za-z0-9_]*",r:0},{b:t+":",r:0},e.CNM,a,r,{b:"\\|[ ]*"+t+"([ ]+"+t+")*[ ]*\\|",rB:!0,e:/\|/,i:/\S/,c:[{b:"(\\|[ ]*)?"+t}]},{b:"\\#\\(",e:"\\)",c:[e.ASM,r,e.CNM,a]}]}})),e.registerLanguage("sml",(function(e){return{aliases:["ml"],k:{keyword:"abstype and andalso as case datatype do else end eqtype exception fn fun functor handle if in include infix infixr let local nonfix of op open orelse raise rec sharing sig signature struct structure then type val with withtype where while",built_in:"array bool char exn int list option order real ref string substring vector unit word",literal:"true false NONE SOME LESS EQUAL GREATER nil"},i:/\/\/|>>/,l:"[a-z_]\\w*!?",c:[{cN:"literal",b:/\[(\|\|)?\]|\(\)/,r:0},e.C("\\(\\*","\\*\\)",{c:["self"]}),{cN:"symbol",b:"'[A-Za-z_](?!')[\\w']*"},{cN:"type",b:"`[A-Z][\\w']*"},{cN:"type",b:"\\b[A-Z][\\w']*",r:0},{b:"[a-z_]\\w*'[\\w']*"},e.inherit(e.ASM,{cN:"string",r:0}),e.inherit(e.QSM,{i:null}),{cN:"number",b:"\\b(0[xX][a-fA-F0-9_]+[Lln]?|0[oO][0-7_]+[Lln]?|0[bB][01_]+[Lln]?|[0-9][0-9_]*([Lln]|(\\.[0-9_]*)?([eE][-+]?[0-9_]+)?)?)",r:0},{b:/[-=]>/}]}})),e.registerLanguage("sqf",(function(e){var t=e.getLanguage("cpp").exports,r={cN:"variable",b:/\b_+[a-zA-Z_]\w*/},a={cN:"title",b:/[a-zA-Z][a-zA-Z0-9]+_fnc_\w*/},i={cN:"string",v:[{b:'"',e:'"',c:[{b:'""',r:0}]},{b:"'",e:"'",c:[{b:"''",r:0}]}]};return{aliases:["sqf"],cI:!0,k:{keyword:"case catch default do else exit exitWith for forEach from if switch then throw to try waitUntil while with",built_in:"abs accTime acos action actionIDs actionKeys actionKeysImages actionKeysNames actionKeysNamesArray actionName actionParams activateAddons activatedAddons activateKey add3DENConnection add3DENEventHandler add3DENLayer addAction addBackpack addBackpackCargo addBackpackCargoGlobal addBackpackGlobal addCamShake addCuratorAddons addCuratorCameraArea addCuratorEditableObjects addCuratorEditingArea addCuratorPoints addEditorObject addEventHandler addGoggles addGroupIcon addHandgunItem addHeadgear addItem addItemCargo addItemCargoGlobal addItemPool addItemToBackpack addItemToUniform addItemToVest addLiveStats addMagazine addMagazineAmmoCargo addMagazineCargo addMagazineCargoGlobal addMagazineGlobal addMagazinePool addMagazines addMagazineTurret addMenu addMenuItem addMissionEventHandler addMPEventHandler addMusicEventHandler addOwnedMine addPlayerScores addPrimaryWeaponItem addPublicVariableEventHandler addRating addResources addScore addScoreSide addSecondaryWeaponItem addSwitchableUnit addTeamMember addToRemainsCollector addUniform addVehicle addVest addWaypoint addWeapon addWeaponCargo addWeaponCargoGlobal addWeaponGlobal addWeaponItem addWeaponPool addWeaponTurret agent agents AGLToASL aimedAtTarget aimPos airDensityRTD airportSide AISFinishHeal alive all3DENEntities allControls allCurators allCutLayers allDead allDeadMen allDisplays allGroups allMapMarkers allMines allMissionObjects allow3DMode allowCrewInImmobile allowCuratorLogicIgnoreAreas allowDamage allowDammage allowFileOperations allowFleeing allowGetIn allowSprint allPlayers allSites allTurrets allUnits allUnitsUAV allVariables ammo and animate animateDoor animateSource animationNames animationPhase animationSourcePhase animationState append apply armoryPoints arrayIntersect asin ASLToAGL ASLToATL assert assignAsCargo assignAsCargoIndex assignAsCommander assignAsDriver assignAsGunner assignAsTurret assignCurator assignedCargo assignedCommander assignedDriver assignedGunner assignedItems assignedTarget assignedTeam assignedVehicle assignedVehicleRole assignItem assignTeam assignToAirport atan atan2 atg ATLToASL attachedObject attachedObjects attachedTo attachObject attachTo attackEnabled backpack backpackCargo backpackContainer backpackItems backpackMagazines backpackSpaceFor behavior benchmark binocular blufor boundingBox boundingBoxReal boundingCenter breakOut breakTo briefingName buildingExit buildingPos buttonAction buttonSetAction cadetMode call callExtension camCommand camCommit camCommitPrepared camCommitted camConstuctionSetParams camCreate camDestroy cameraEffect cameraEffectEnableHUD cameraInterest cameraOn cameraView campaignConfigFile camPreload camPreloaded camPrepareBank camPrepareDir camPrepareDive camPrepareFocus camPrepareFov camPrepareFovRange camPreparePos camPrepareRelPos camPrepareTarget camSetBank camSetDir camSetDive camSetFocus camSetFov camSetFovRange camSetPos camSetRelPos camSetTarget camTarget camUseNVG canAdd canAddItemToBackpack canAddItemToUniform canAddItemToVest cancelSimpleTaskDestination canFire canMove canSlingLoad canStand canSuspend canUnloadInCombat canVehicleCargo captive captiveNum cbChecked cbSetChecked ceil channelEnabled cheatsEnabled checkAIFeature checkVisibility civilian className clearAllItemsFromBackpack clearBackpackCargo clearBackpackCargoGlobal clearGroupIcons clearItemCargo clearItemCargoGlobal clearItemPool clearMagazineCargo clearMagazineCargoGlobal clearMagazinePool clearOverlay clearRadio clearWeaponCargo clearWeaponCargoGlobal clearWeaponPool clientOwner closeDialog closeDisplay closeOverlay collapseObjectTree collect3DENHistory combatMode commandArtilleryFire commandChat commander commandFire commandFollow commandFSM commandGetOut commandingMenu commandMove commandRadio commandStop commandSuppressiveFire commandTarget commandWatch comment commitOverlay compile compileFinal completedFSM composeText configClasses configFile configHierarchy configName configNull configProperties configSourceAddonList configSourceMod configSourceModList connectTerminalToUAV controlNull controlsGroupCtrl copyFromClipboard copyToClipboard copyWaypoints cos count countEnemy countFriendly countSide countType countUnknown create3DENComposition create3DENEntity createAgent createCenter createDialog createDiaryLink createDiaryRecord createDiarySubject createDisplay createGearDialog createGroup createGuardedPoint createLocation createMarker createMarkerLocal createMenu createMine createMissionDisplay createMPCampaignDisplay createSimpleObject createSimpleTask createSite createSoundSource createTask createTeam createTrigger createUnit createVehicle createVehicleCrew createVehicleLocal crew ctrlActivate ctrlAddEventHandler ctrlAngle ctrlAutoScrollDelay ctrlAutoScrollRewind ctrlAutoScrollSpeed ctrlChecked ctrlClassName ctrlCommit ctrlCommitted ctrlCreate ctrlDelete ctrlEnable ctrlEnabled ctrlFade ctrlHTMLLoaded ctrlIDC ctrlIDD ctrlMapAnimAdd ctrlMapAnimClear ctrlMapAnimCommit ctrlMapAnimDone ctrlMapCursor ctrlMapMouseOver ctrlMapScale ctrlMapScreenToWorld ctrlMapWorldToScreen ctrlModel ctrlModelDirAndUp ctrlModelScale ctrlParent ctrlParentControlsGroup ctrlPosition ctrlRemoveAllEventHandlers ctrlRemoveEventHandler ctrlScale ctrlSetActiveColor ctrlSetAngle ctrlSetAutoScrollDelay ctrlSetAutoScrollRewind ctrlSetAutoScrollSpeed ctrlSetBackgroundColor ctrlSetChecked ctrlSetEventHandler ctrlSetFade ctrlSetFocus ctrlSetFont ctrlSetFontH1 ctrlSetFontH1B ctrlSetFontH2 ctrlSetFontH2B ctrlSetFontH3 ctrlSetFontH3B ctrlSetFontH4 ctrlSetFontH4B ctrlSetFontH5 ctrlSetFontH5B ctrlSetFontH6 ctrlSetFontH6B ctrlSetFontHeight ctrlSetFontHeightH1 ctrlSetFontHeightH2 ctrlSetFontHeightH3 ctrlSetFontHeightH4 ctrlSetFontHeightH5 ctrlSetFontHeightH6 ctrlSetFontHeightSecondary ctrlSetFontP ctrlSetFontPB ctrlSetFontSecondary ctrlSetForegroundColor ctrlSetModel ctrlSetModelDirAndUp ctrlSetModelScale ctrlSetPosition ctrlSetScale ctrlSetStructuredText ctrlSetText ctrlSetTextColor ctrlSetTooltip ctrlSetTooltipColorBox ctrlSetTooltipColorShade ctrlSetTooltipColorText ctrlShow ctrlShown ctrlText ctrlTextHeight ctrlType ctrlVisible curatorAddons curatorCamera curatorCameraArea curatorCameraAreaCeiling curatorCoef curatorEditableObjects curatorEditingArea curatorEditingAreaType curatorMouseOver curatorPoints curatorRegisteredObjects curatorSelected curatorWaypointCost current3DENOperation currentChannel currentCommand currentMagazine currentMagazineDetail currentMagazineDetailTurret currentMagazineTurret currentMuzzle currentNamespace currentTask currentTasks currentThrowable currentVisionMode currentWaypoint currentWeapon currentWeaponMode currentWeaponTurret currentZeroing cursorObject cursorTarget customChat customRadio cutFadeOut cutObj cutRsc cutText damage date dateToNumber daytime deActivateKey debriefingText debugFSM debugLog deg delete3DENEntities deleteAt deleteCenter deleteCollection deleteEditorObject deleteGroup deleteIdentity deleteLocation deleteMarker deleteMarkerLocal deleteRange deleteResources deleteSite deleteStatus deleteTeam deleteVehicle deleteVehicleCrew deleteWaypoint detach detectedMines diag_activeMissionFSMs diag_activeScripts diag_activeSQFScripts diag_activeSQSScripts diag_captureFrame diag_captureSlowFrame diag_codePerformance diag_drawMode diag_enable diag_enabled diag_fps diag_fpsMin diag_frameNo diag_list diag_log diag_logSlowFrame diag_mergeConfigFile diag_recordTurretLimits diag_tickTime diag_toggle dialog diarySubjectExists didJIP didJIPOwner difficulty difficultyEnabled difficultyEnabledRTD difficultyOption direction directSay disableAI disableCollisionWith disableConversation disableDebriefingStats disableNVGEquipment disableRemoteSensors disableSerialization disableTIEquipment disableUAVConnectability disableUserInput displayAddEventHandler displayCtrl displayNull displayParent displayRemoveAllEventHandlers displayRemoveEventHandler displaySetEventHandler dissolveTeam distance distance2D distanceSqr distributionRegion do3DENAction doArtilleryFire doFire doFollow doFSM doGetOut doMove doorPhase doStop doSuppressiveFire doTarget doWatch drawArrow drawEllipse drawIcon drawIcon3D drawLine drawLine3D drawLink drawLocation drawPolygon drawRectangle driver drop east echo edit3DENMissionAttributes editObject editorSetEventHandler effectiveCommander emptyPositions enableAI enableAIFeature enableAimPrecision enableAttack enableAudioFeature enableCamShake enableCaustics enableChannel enableCollisionWith enableCopilot enableDebriefingStats enableDiagLegend enableEndDialog enableEngineArtillery enableEnvironment enableFatigue enableGunLights enableIRLasers enableMimics enablePersonTurret enableRadio enableReload enableRopeAttach enableSatNormalOnDetail enableSaving enableSentences enableSimulation enableSimulationGlobal enableStamina enableTeamSwitch enableUAVConnectability enableUAVWaypoints enableVehicleCargo endLoadingScreen endMission engineOn enginesIsOnRTD enginesRpmRTD enginesTorqueRTD entities estimatedEndServerTime estimatedTimeLeft evalObjectArgument everyBackpack everyContainer exec execEditorScript execFSM execVM exp expectedDestination exportJIPMessages eyeDirection eyePos face faction fadeMusic fadeRadio fadeSound fadeSpeech failMission fillWeaponsFromPool find findCover findDisplay findEditorObject findEmptyPosition findEmptyPositionReady findNearestEnemy finishMissionInit finite fire fireAtTarget firstBackpack flag flagOwner flagSide flagTexture fleeing floor flyInHeight flyInHeightASL fog fogForecast fogParams forceAddUniform forcedMap forceEnd forceMap forceRespawn forceSpeed forceWalk forceWeaponFire forceWeatherChange forEachMember forEachMemberAgent forEachMemberTeam format formation formationDirection formationLeader formationMembers formationPosition formationTask formatText formLeader freeLook fromEditor fuel fullCrew gearIDCAmmoCount gearSlotAmmoCount gearSlotData get3DENActionState get3DENAttribute get3DENCamera get3DENConnections get3DENEntity get3DENEntityID get3DENGrid get3DENIconsVisible get3DENLayerEntities get3DENLinesVisible get3DENMissionAttribute get3DENMouseOver get3DENSelected getAimingCoef getAllHitPointsDamage getAllOwnedMines getAmmoCargo getAnimAimPrecision getAnimSpeedCoef getArray getArtilleryAmmo getArtilleryComputerSettings getArtilleryETA getAssignedCuratorLogic getAssignedCuratorUnit getBackpackCargo getBleedingRemaining getBurningValue getCameraViewDirection getCargoIndex getCenterOfMass getClientState getClientStateNumber getConnectedUAV getCustomAimingCoef getDammage getDescription getDir getDirVisual getDLCs getEditorCamera getEditorMode getEditorObjectScope getElevationOffset getFatigue getFriend getFSMVariable getFuelCargo getGroupIcon getGroupIconParams getGroupIcons getHideFrom getHit getHitIndex getHitPointDamage getItemCargo getMagazineCargo getMarkerColor getMarkerPos getMarkerSize getMarkerType getMass getMissionConfig getMissionConfigValue getMissionDLCs getMissionLayerEntities getModelInfo getMousePosition getNumber getObjectArgument getObjectChildren getObjectDLC getObjectMaterials getObjectProxy getObjectTextures getObjectType getObjectViewDistance getOxygenRemaining getPersonUsedDLCs getPilotCameraDirection getPilotCameraPosition getPilotCameraRotation getPilotCameraTarget getPlayerChannel getPlayerScores getPlayerUID getPos getPosASL getPosASLVisual getPosASLW getPosATL getPosATLVisual getPosVisual getPosWorld getRelDir getRelPos getRemoteSensorsDisabled getRepairCargo getResolution getShadowDistance getShotParents getSlingLoad getSpeed getStamina getStatValue getSuppression getTerrainHeightASL getText getUnitLoadout getUnitTrait getVariable getVehicleCargo getWeaponCargo getWeaponSway getWPPos glanceAt globalChat globalRadio goggles goto group groupChat groupFromNetId groupIconSelectable groupIconsVisible groupId groupOwner groupRadio groupSelectedUnits groupSelectUnit grpNull gunner gusts halt handgunItems handgunMagazine handgunWeapon handsHit hasInterface hasPilotCamera hasWeapon hcAllGroups hcGroupParams hcLeader hcRemoveAllGroups hcRemoveGroup hcSelected hcSelectGroup hcSetGroup hcShowBar hcShownBar headgear hideBody hideObject hideObjectGlobal hideSelection hint hintC hintCadet hintSilent hmd hostMission htmlLoad HUDMovementLevels humidity image importAllGroups importance in inArea inAreaArray incapacitatedState independent inflame inflamed inGameUISetEventHandler inheritsFrom initAmbientLife inPolygon inputAction inRangeOfArtillery insertEditorObject intersect is3DEN is3DENMultiplayer isAbleToBreathe isAgent isArray isAutoHoverOn isAutonomous isAutotest isBleeding isBurning isClass isCollisionLightOn isCopilotEnabled isDedicated isDLCAvailable isEngineOn isEqualTo isEqualType isEqualTypeAll isEqualTypeAny isEqualTypeArray isEqualTypeParams isFilePatchingEnabled isFlashlightOn isFlatEmpty isForcedWalk isFormationLeader isHidden isInRemainsCollector isInstructorFigureEnabled isIRLaserOn isKeyActive isKindOf isLightOn isLocalized isManualFire isMarkedForCollection isMultiplayer isMultiplayerSolo isNil isNull isNumber isObjectHidden isObjectRTD isOnRoad isPipEnabled isPlayer isRealTime isRemoteExecuted isRemoteExecutedJIP isServer isShowing3DIcons isSprintAllowed isStaminaEnabled isSteamMission isStreamFriendlyUIEnabled isText isTouchingGround isTurnedOut isTutHintsEnabled isUAVConnectable isUAVConnected isUniformAllowed isVehicleCargo isWalking isWeaponDeployed isWeaponRested itemCargo items itemsWithMagazines join joinAs joinAsSilent joinSilent joinString kbAddDatabase kbAddDatabaseTargets kbAddTopic kbHasTopic kbReact kbRemoveTopic kbTell kbWasSaid keyImage keyName knowsAbout land landAt landResult language laserTarget lbAdd lbClear lbColor lbCurSel lbData lbDelete lbIsSelected lbPicture lbSelection lbSetColor lbSetCurSel lbSetData lbSetPicture lbSetPictureColor lbSetPictureColorDisabled lbSetPictureColorSelected lbSetSelectColor lbSetSelectColorRight lbSetSelected lbSetTooltip lbSetValue lbSize lbSort lbSortByValue lbText lbValue leader leaderboardDeInit leaderboardGetRows leaderboardInit leaveVehicle libraryCredits libraryDisclaimers lifeState lightAttachObject lightDetachObject lightIsOn lightnings limitSpeed linearConversion lineBreak lineIntersects lineIntersectsObjs lineIntersectsSurfaces lineIntersectsWith linkItem list listObjects ln lnbAddArray lnbAddColumn lnbAddRow lnbClear lnbColor lnbCurSelRow lnbData lnbDeleteColumn lnbDeleteRow lnbGetColumnsPosition lnbPicture lnbSetColor lnbSetColumnsPos lnbSetCurSelRow lnbSetData lnbSetPicture lnbSetText lnbSetValue lnbSize lnbText lnbValue load loadAbs loadBackpack loadFile loadGame loadIdentity loadMagazine loadOverlay loadStatus loadUniform loadVest local localize locationNull locationPosition lock lockCameraTo lockCargo lockDriver locked lockedCargo lockedDriver lockedTurret lockIdentity lockTurret lockWP log logEntities logNetwork logNetworkTerminate lookAt lookAtPos magazineCargo magazines magazinesAllTurrets magazinesAmmo magazinesAmmoCargo magazinesAmmoFull magazinesDetail magazinesDetailBackpack magazinesDetailUniform magazinesDetailVest magazinesTurret magazineTurretAmmo mapAnimAdd mapAnimClear mapAnimCommit mapAnimDone mapCenterOnCamera mapGridPosition markAsFinishedOnSteam markerAlpha markerBrush markerColor markerDir markerPos markerShape markerSize markerText markerType max members menuAction menuAdd menuChecked menuClear menuCollapse menuData menuDelete menuEnable menuEnabled menuExpand menuHover menuPicture menuSetAction menuSetCheck menuSetData menuSetPicture menuSetValue menuShortcut menuShortcutText menuSize menuSort menuText menuURL menuValue min mineActive mineDetectedBy missionConfigFile missionDifficulty missionName missionNamespace missionStart missionVersion mod modelToWorld modelToWorldVisual modParams moonIntensity moonPhase morale move move3DENCamera moveInAny moveInCargo moveInCommander moveInDriver moveInGunner moveInTurret moveObjectToEnd moveOut moveTime moveTo moveToCompleted moveToFailed musicVolume name nameSound nearEntities nearestBuilding nearestLocation nearestLocations nearestLocationWithDubbing nearestObject nearestObjects nearestTerrainObjects nearObjects nearObjectsReady nearRoads nearSupplies nearTargets needReload netId netObjNull newOverlay nextMenuItemIndex nextWeatherChange nMenuItems not numberToDate objectCurators objectFromNetId objectParent objNull objStatus onBriefingGroup onBriefingNotes onBriefingPlan onBriefingTeamSwitch onCommandModeChanged onDoubleClick onEachFrame onGroupIconClick onGroupIconOverEnter onGroupIconOverLeave onHCGroupSelectionChanged onMapSingleClick onPlayerConnected onPlayerDisconnected onPreloadFinished onPreloadStarted onShowNewObject onTeamSwitch openCuratorInterface openDLCPage openMap openYoutubeVideo opfor or orderGetIn overcast overcastForecast owner param params parseNumber parseText parsingNamespace particlesQuality pi pickWeaponPool pitch pixelGrid pixelGridBase pixelGridNoUIScale pixelH pixelW playableSlotsNumber playableUnits playAction playActionNow player playerRespawnTime playerSide playersNumber playGesture playMission playMove playMoveNow playMusic playScriptedMission playSound playSound3D position positionCameraToWorld posScreenToWorld posWorldToScreen ppEffectAdjust ppEffectCommit ppEffectCommitted ppEffectCreate ppEffectDestroy ppEffectEnable ppEffectEnabled ppEffectForceInNVG precision preloadCamera preloadObject preloadSound preloadTitleObj preloadTitleRsc preprocessFile preprocessFileLineNumbers primaryWeapon primaryWeaponItems primaryWeaponMagazine priority private processDiaryLink productVersion profileName profileNamespace profileNameSteam progressLoadingScreen progressPosition progressSetPosition publicVariable publicVariableClient publicVariableServer pushBack pushBackUnique putWeaponPool queryItemsPool queryMagazinePool queryWeaponPool rad radioChannelAdd radioChannelCreate radioChannelRemove radioChannelSetCallSign radioChannelSetLabel radioVolume rain rainbow random rank rankId rating rectangular registeredTasks registerTask reload reloadEnabled remoteControl remoteExec remoteExecCall remove3DENConnection remove3DENEventHandler remove3DENLayer removeAction removeAll3DENEventHandlers removeAllActions removeAllAssignedItems removeAllContainers removeAllCuratorAddons removeAllCuratorCameraAreas removeAllCuratorEditingAreas removeAllEventHandlers removeAllHandgunItems removeAllItems removeAllItemsWithMagazines removeAllMissionEventHandlers removeAllMPEventHandlers removeAllMusicEventHandlers removeAllOwnedMines removeAllPrimaryWeaponItems removeAllWeapons removeBackpack removeBackpackGlobal removeCuratorAddons removeCuratorCameraArea removeCuratorEditableObjects removeCuratorEditingArea removeDrawIcon removeDrawLinks removeEventHandler removeFromRemainsCollector removeGoggles removeGroupIcon removeHandgunItem removeHeadgear removeItem removeItemFromBackpack removeItemFromUniform removeItemFromVest removeItems removeMagazine removeMagazineGlobal removeMagazines removeMagazinesTurret removeMagazineTurret removeMenuItem removeMissionEventHandler removeMPEventHandler removeMusicEventHandler removeOwnedMine removePrimaryWeaponItem removeSecondaryWeaponItem removeSimpleTask removeSwitchableUnit removeTeamMember removeUniform removeVest removeWeapon removeWeaponGlobal removeWeaponTurret requiredVersion resetCamShake resetSubgroupDirection resistance resize resources respawnVehicle restartEditorCamera reveal revealMine reverse reversedMouseY roadAt roadsConnectedTo roleDescription ropeAttachedObjects ropeAttachedTo ropeAttachEnabled ropeAttachTo ropeCreate ropeCut ropeDestroy ropeDetach ropeEndPosition ropeLength ropes ropeUnwind ropeUnwound rotorsForcesRTD rotorsRpmRTD round runInitScript safeZoneH safeZoneW safeZoneWAbs safeZoneX safeZoneXAbs safeZoneY save3DENInventory saveGame saveIdentity saveJoysticks saveOverlay saveProfileNamespace saveStatus saveVar savingEnabled say say2D say3D scopeName score scoreSide screenshot screenToWorld scriptDone scriptName scriptNull scudState secondaryWeapon secondaryWeaponItems secondaryWeaponMagazine select selectBestPlaces selectDiarySubject selectedEditorObjects selectEditorObject selectionNames selectionPosition selectLeader selectMax selectMin selectNoPlayer selectPlayer selectRandom selectWeapon selectWeaponTurret sendAUMessage sendSimpleCommand sendTask sendTaskResult sendUDPMessage serverCommand serverCommandAvailable serverCommandExecutable serverName serverTime set set3DENAttribute set3DENAttributes set3DENGrid set3DENIconsVisible set3DENLayer set3DENLinesVisible set3DENMissionAttributes set3DENModelsVisible set3DENObjectType set3DENSelected setAccTime setAirportSide setAmmo setAmmoCargo setAnimSpeedCoef setAperture setApertureNew setArmoryPoints setAttributes setAutonomous setbehavior setBleedingRemaining setCameraInterest setCamShakeDefParams setCamShakeParams setCamUseTi setCaptive setCenterOfMass setCollisionLight setCombatMode setCompassOscillation setCuratorCameraAreaCeiling setCuratorCoef setCuratorEditingAreaType setCuratorWaypointCost setCurrentChannel setCurrentTask setCurrentWaypoint setCustomAimCoef setDamage setDammage setDate setDebriefingText setDefaultCamera setDestination setDetailMapBlendPars setDir setDirection setDrawIcon setDropInterval setEditorMode setEditorObjectScope setEffectCondition setFace setFaceAnimation setFatigue setFlagOwner setFlagSide setFlagTexture setFog setFormation setFormationTask setFormDir setFriend setFromEditor setFSMVariable setFuel setFuelCargo setGroupIcon setGroupIconParams setGroupIconsSelectable setGroupIconsVisible setGroupId setGroupIdGlobal setGroupOwner setGusts setHideBehind setHit setHitIndex setHitPointDamage setHorizonParallaxCoef setHUDMovementLevels setIdentity setImportance setLeader setLightAmbient setLightAttenuation setLightBrightness setLightColor setLightDayLight setLightFlareMaxDistance setLightFlareSize setLightIntensity setLightnings setLightUseFlare setLocalWindParams setMagazineTurretAmmo setMarkerAlpha setMarkerAlphaLocal setMarkerBrush setMarkerBrushLocal setMarkerColor setMarkerColorLocal setMarkerDir setMarkerDirLocal setMarkerPos setMarkerPosLocal setMarkerShape setMarkerShapeLocal setMarkerSize setMarkerSizeLocal setMarkerText setMarkerTextLocal setMarkerType setMarkerTypeLocal setMass setMimic setMousePosition setMusicEffect setMusicEventHandler setName setNameSound setObjectArguments setObjectMaterial setObjectMaterialGlobal setObjectProxy setObjectTexture setObjectTextureGlobal setObjectViewDistance setOvercast setOwner setOxygenRemaining setParticleCircle setParticleClass setParticleFire setParticleParams setParticleRandom setPilotCameraDirection setPilotCameraRotation setPilotCameraTarget setPilotLight setPiPEffect setPitch setPlayable setPlayerRespawnTime setPos setPosASL setPosASL2 setPosASLW setPosATL setPosition setPosWorld setRadioMsg setRain setRainbow setRandomLip setRank setRectangular setRepairCargo setShadowDistance setShotParents setSide setSimpleTaskAlwaysVisible setSimpleTaskCustomData setSimpleTaskDescription setSimpleTaskDestination setSimpleTaskTarget setSimpleTaskType setSimulWeatherLayers setSize setSkill setSlingLoad setSoundEffect setSpeaker setSpeech setSpeedMode setStamina setStaminaScheme setStatValue setSuppression setSystemOfUnits setTargetAge setTaskResult setTaskState setTerrainGrid setText setTimeMultiplier setTitleEffect setTriggerActivation setTriggerArea setTriggerStatements setTriggerText setTriggerTimeout setTriggerType setType setUnconscious setUnitAbility setUnitLoadout setUnitPos setUnitPosWeak setUnitRank setUnitRecoilCoefficient setUnitTrait setUnloadInCombat setUserActionText setVariable setVectorDir setVectorDirAndUp setVectorUp setVehicleAmmo setVehicleAmmoDef setVehicleArmor setVehicleCargo setVehicleId setVehicleLock setVehiclePosition setVehicleTiPars setVehicleVarName setVelocity setVelocityTransformation setViewDistance setVisibleIfTreeCollapsed setWaves setWaypointbehavior setWaypointCombatMode setWaypointCompletionRadius setWaypointDescription setWaypointForcebehavior setWaypointFormation setWaypointHousePosition setWaypointLoiterRadius setWaypointLoiterType setWaypointName setWaypointPosition setWaypointScript setWaypointSpeed setWaypointStatements setWaypointTimeout setWaypointType setWaypointVisible setWeaponReloadingTime setWind setWindDir setWindForce setWindStr setWPPos show3DIcons showChat showCinemaBorder showCommandingMenu showCompass showCuratorCompass showGPS showHUD showLegend showMap shownArtilleryComputer shownChat shownCompass shownCuratorCompass showNewEditorObject shownGPS shownHUD shownMap shownPad shownRadio shownScoretable shownUAVFeed shownWarrant shownWatch showPad showRadio showScoretable showSubtitles showUAVFeed showWarrant showWatch showWaypoint showWaypoints side sideAmbientLife sideChat sideEmpty sideEnemy sideFriendly sideLogic sideRadio sideUnknown simpleTasks simulationEnabled simulCloudDensity simulCloudOcclusion simulInClouds simulWeatherSync sin size sizeOf skill skillFinal skipTime sleep sliderPosition sliderRange sliderSetPosition sliderSetRange sliderSetSpeed sliderSpeed slingLoadAssistantShown soldierMagazines someAmmo sort soundVolume spawn speaker speed speedMode splitString sqrt squadParams stance startLoadingScreen step stop stopEngineRTD stopped str sunOrMoon supportInfo suppressFor surfaceIsWater surfaceNormal surfaceType swimInDepth switchableUnits switchAction switchCamera switchGesture switchLight switchMove synchronizedObjects synchronizedTriggers synchronizedWaypoints synchronizeObjectsAdd synchronizeObjectsRemove synchronizeTrigger synchronizeWaypoint systemChat systemOfUnits tan targetKnowledge targetsAggregate targetsQuery taskAlwaysVisible taskChildren taskCompleted taskCustomData taskDescription taskDestination taskHint taskMarkerOffset taskNull taskParent taskResult taskState taskType teamMember teamMemberNull teamName teams teamSwitch teamSwitchEnabled teamType terminate terrainIntersect terrainIntersectASL text textLog textLogFormat tg time timeMultiplier titleCut titleFadeOut titleObj titleRsc titleText toArray toFixed toLower toString toUpper triggerActivated triggerActivation triggerArea triggerAttachedVehicle triggerAttachObject triggerAttachVehicle triggerStatements triggerText triggerTimeout triggerTimeoutCurrent triggerType turretLocal turretOwner turretUnit tvAdd tvClear tvCollapse tvCount tvCurSel tvData tvDelete tvExpand tvPicture tvSetCurSel tvSetData tvSetPicture tvSetPictureColor tvSetPictureColorDisabled tvSetPictureColorSelected tvSetPictureRight tvSetPictureRightColor tvSetPictureRightColorDisabled tvSetPictureRightColorSelected tvSetText tvSetTooltip tvSetValue tvSort tvSortByValue tvText tvTooltip tvValue type typeName typeOf UAVControl uiNamespace uiSleep unassignCurator unassignItem unassignTeam unassignVehicle underwater uniform uniformContainer uniformItems uniformMagazines unitAddons unitAimPosition unitAimPositionVisual unitBackpack unitIsUAV unitPos unitReady unitRecoilCoefficient units unitsBelowHeight unlinkItem unlockAchievement unregisterTask updateDrawIcon updateMenuItem updateObjectTree useAISteeringComponent useAudioTimeForMoves vectorAdd vectorCos vectorCrossProduct vectorDiff vectorDir vectorDirVisual vectorDistance vectorDistanceSqr vectorDotProduct vectorFromTo vectorMagnitude vectorMagnitudeSqr vectorMultiply vectorNormalized vectorUp vectorUpVisual vehicle vehicleCargoEnabled vehicleChat vehicleRadio vehicles vehicleVarName velocity velocityModelSpace verifySignature vest vestContainer vestItems vestMagazines viewDistance visibleCompass visibleGPS visibleMap visiblePosition visiblePositionASL visibleScoretable visibleWatch waves waypointAttachedObject waypointAttachedVehicle waypointAttachObject waypointAttachVehicle waypointbehavior waypointCombatMode waypointCompletionRadius waypointDescription waypointForcebehavior waypointFormation waypointHousePosition waypointLoiterRadius waypointLoiterType waypointName waypointPosition waypoints waypointScript waypointsEnabledUAV waypointShow waypointSpeed waypointStatements waypointTimeout waypointTimeoutCurrent waypointType waypointVisible weaponAccessories weaponAccessoriesCargo weaponCargo weaponDirection weaponInertia weaponLowered weapons weaponsItems weaponsItemsCargo weaponState weaponsTurret weightRTD west WFSideText wind",literal:"true false nil"},c:[e.CLCM,e.CBCM,e.NM,r,a,i,t.preprocessor],i:/#/}})),e.registerLanguage("sql",(function(e){var t=e.C("--","$");return{cI:!0,i:/[<>{}*#]/,c:[{bK:"begin end start commit rollback savepoint lock alter create drop rename call delete do handler insert load replace select truncate update set show pragma grant merge describe use explain help declare prepare execute deallocate release unlock purge reset change stop analyze cache flush optimize repair kill install uninstall checksum restore check backup revoke comment",e:/;/,eW:!0,l:/[\w\.]+/,k:{keyword:"abort abs absolute acc acce accep accept access accessed accessible account acos action activate add addtime admin administer advanced advise aes_decrypt aes_encrypt after agent aggregate ali alia alias allocate allow alter always analyze ancillary and any anydata anydataset anyschema anytype apply archive archived archivelog are as asc ascii asin assembly assertion associate asynchronous at atan atn2 attr attri attrib attribu attribut attribute attributes audit authenticated authentication authid authors auto autoallocate autodblink autoextend automatic availability avg backup badfile basicfile before begin beginning benchmark between bfile bfile_base big bigfile bin binary_double binary_float binlog bit_and bit_count bit_length bit_or bit_xor bitmap blob_base block blocksize body both bound buffer_cache buffer_pool build bulk by byte byteordermark bytes cache caching call calling cancel capacity cascade cascaded case cast catalog category ceil ceiling chain change changed char_base char_length character_length characters characterset charindex charset charsetform charsetid check checksum checksum_agg child choose chr chunk class cleanup clear client clob clob_base clone close cluster_id cluster_probability cluster_set clustering coalesce coercibility col collate collation collect colu colum column column_value columns columns_updated comment commit compact compatibility compiled complete composite_limit compound compress compute concat concat_ws concurrent confirm conn connec connect connect_by_iscycle connect_by_isleaf connect_by_root connect_time connection consider consistent constant constraint constraints constructor container content contents context contributors controlfile conv convert convert_tz corr corr_k corr_s corresponding corruption cos cost count count_big counted covar_pop covar_samp cpu_per_call cpu_per_session crc32 create creation critical cross cube cume_dist curdate current current_date current_time current_timestamp current_user cursor curtime customdatum cycle data database databases datafile datafiles datalength date_add date_cache date_format date_sub dateadd datediff datefromparts datename datepart datetime2fromparts day day_to_second dayname dayofmonth dayofweek dayofyear days db_role_change dbtimezone ddl deallocate declare decode decompose decrement decrypt deduplicate def defa defau defaul default defaults deferred defi defin define degrees delayed delegate delete delete_all delimited demand dense_rank depth dequeue des_decrypt des_encrypt des_key_file desc descr descri describ describe descriptor deterministic diagnostics difference dimension direct_load directory disable disable_all disallow disassociate discardfile disconnect diskgroup distinct distinctrow distribute distributed div do document domain dotnet double downgrade drop dumpfile duplicate duration each edition editionable editions element ellipsis else elsif elt empty enable enable_all enclosed encode encoding encrypt end end-exec endian enforced engine engines enqueue enterprise entityescaping eomonth error errors escaped evalname evaluate event eventdata events except exception exceptions exchange exclude excluding execu execut execute exempt exists exit exp expire explain export export_set extended extent external external_1 external_2 externally extract failed failed_login_attempts failover failure far fast feature_set feature_value fetch field fields file file_name_convert filesystem_like_logging final finish first first_value fixed flash_cache flashback floor flush following follows for forall force form forma format found found_rows freelist freelists freepools fresh from from_base64 from_days ftp full function general generated get get_format get_lock getdate getutcdate global global_name globally go goto grant grants greatest group group_concat group_id grouping grouping_id groups gtid_subtract guarantee guard handler hash hashkeys having hea head headi headin heading heap help hex hierarchy high high_priority hosts hour http id ident_current ident_incr ident_seed identified identity idle_time if ifnull ignore iif ilike ilm immediate import in include including increment index indexes indexing indextype indicator indices inet6_aton inet6_ntoa inet_aton inet_ntoa infile initial initialized initially initrans inmemory inner innodb input insert install instance instantiable instr interface interleaved intersect into invalidate invisible is is_free_lock is_ipv4 is_ipv4_compat is_not is_not_null is_used_lock isdate isnull isolation iterate java join json json_exists keep keep_duplicates key keys kill language large last last_day last_insert_id last_value lax lcase lead leading least leaves left len length length less level levels library like like2 like4 likec limit lines link list listagg little ln load load_file lob lobs local localtime localtimestamp locate locator lock locked log log10 log2 logfile logfiles logging logical logical_reads_per_call logoff logon logs long loop low low_priority lower lpad lrtrim ltrim main make_set makedate maketime managed management manual map mapping mask master master_pos_wait match matched materialized max maxextents maximize maxinstances maxlen maxlogfiles maxloghistory maxlogmembers maxsize maxtrans md5 measures median medium member memcompress memory merge microsecond mid migration min minextents minimum mining minus minute minvalue missing mod mode model modification modify module monitoring month months mount move movement multiset mutex name name_const names nan national native natural nav nchar nclob nested never new newline next nextval no no_write_to_binlog noarchivelog noaudit nobadfile nocheck nocompress nocopy nocycle nodelay nodiscardfile noentityescaping noguarantee nokeep nologfile nomapping nomaxvalue nominimize nominvalue nomonitoring none noneditionable nonschema noorder nopr nopro noprom nopromp noprompt norely noresetlogs noreverse normal norowdependencies noschemacheck noswitch not nothing notice notrim novalidate now nowait nth_value nullif nulls num numb numbe nvarchar nvarchar2 object ocicoll ocidate ocidatetime ociduration ociinterval ociloblocator ocinumber ociref ocirefcursor ocirowid ocistring ocitype oct octet_length of off offline offset oid oidindex old on online only opaque open operations operator optimal optimize option optionally or oracle oracle_date oradata ord ordaudio orddicom orddoc order ordimage ordinality ordvideo organization orlany orlvary out outer outfile outline output over overflow overriding package pad parallel parallel_enable parameters parent parse partial partition partitions pascal passing password password_grace_time password_lock_time password_reuse_max password_reuse_time password_verify_function patch path patindex pctincrease pctthreshold pctused pctversion percent percent_rank percentile_cont percentile_disc performance period period_add period_diff permanent physical pi pipe pipelined pivot pluggable plugin policy position post_transaction pow power pragma prebuilt precedes preceding precision prediction prediction_cost prediction_details prediction_probability prediction_set prepare present preserve prior priority private private_sga privileges procedural procedure procedure_analyze processlist profiles project prompt protection public publishingservername purge quarter query quick quiesce quota quotename radians raise rand range rank raw read reads readsize rebuild record records recover recovery recursive recycle redo reduced ref reference referenced references referencing refresh regexp_like register regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy reject rekey relational relative relaylog release release_lock relies_on relocate rely rem remainder rename repair repeat replace replicate replication required reset resetlogs resize resource respect restore restricted result result_cache resumable resume retention return returning returns reuse reverse revoke right rlike role roles rollback rolling rollup round row row_count rowdependencies rowid rownum rows rtrim rules safe salt sample save savepoint sb1 sb2 sb4 scan schema schemacheck scn scope scroll sdo_georaster sdo_topo_geometry search sec_to_time second section securefile security seed segment select self sequence sequential serializable server servererror session session_user sessions_per_user set sets settings sha sha1 sha2 share shared shared_pool short show shrink shutdown si_averagecolor si_colorhistogram si_featurelist si_positionalcolor si_stillimage si_texture siblings sid sign sin size size_t sizes skip slave sleep smalldatetimefromparts smallfile snapshot some soname sort soundex source space sparse spfile split sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_small_result sql_variant_property sqlcode sqldata sqlerror sqlname sqlstate sqrt square standalone standby start starting startup statement static statistics stats_binomial_test stats_crosstab stats_ks_test stats_mode stats_mw_test stats_one_way_anova stats_t_test_ stats_t_test_indep stats_t_test_one stats_t_test_paired stats_wsr_test status std stddev stddev_pop stddev_samp stdev stop storage store stored str str_to_date straight_join strcmp strict string struct stuff style subdate subpartition subpartitions substitutable substr substring subtime subtring_index subtype success sum suspend switch switchoffset switchover sync synchronous synonym sys sys_xmlagg sysasm sysaux sysdate sysdatetimeoffset sysdba sysoper system system_user sysutcdatetime table tables tablespace tan tdo template temporary terminated tertiary_weights test than then thread through tier ties time time_format time_zone timediff timefromparts timeout timestamp timestampadd timestampdiff timezone_abbr timezone_minute timezone_region to to_base64 to_date to_days to_seconds todatetimeoffset trace tracking transaction transactional translate translation treat trigger trigger_nestlevel triggers trim truncate try_cast try_convert try_parse type ub1 ub2 ub4 ucase unarchived unbounded uncompress under undo unhex unicode uniform uninstall union unique unix_timestamp unknown unlimited unlock unpivot unrecoverable unsafe unsigned until untrusted unusable unused update updated upgrade upped upper upsert url urowid usable usage use use_stored_outlines user user_data user_resources users using utc_date utc_timestamp uuid uuid_short validate validate_password_strength validation valist value values var var_samp varcharc vari varia variab variabl variable variables variance varp varraw varrawc varray verify version versions view virtual visible void wait wallet warning warnings week weekday weekofyear wellformed when whene whenev wheneve whenever where while whitespace with within without work wrapped xdb xml xmlagg xmlattributes xmlcast xmlcolattval xmlelement xmlexists xmlforest xmlindex xmlnamespaces xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltype xor year year_to_month years yearweek",literal:"true false null",built_in:"array bigint binary bit blob boolean char character date dec decimal float int int8 integer interval number numeric real record serial serial8 smallint text varchar varying void"},c:[{cN:"string",b:"'",e:"'",c:[e.BE,{b:"''"}]},{cN:"string",b:'"',e:'"',c:[e.BE,{b:'""'}]},{cN:"string",b:"`",e:"`",c:[e.BE]},e.CNM,e.CBCM,t]},e.CBCM,t]}})),e.registerLanguage("stan",(function(e){return{c:[e.HCM,e.CLCM,e.CBCM,{b:e.UIR,l:e.UIR,k:{name:"for in while repeat until if then else",symbol:"bernoulli bernoulli_logit binomial binomial_logit beta_binomial hypergeometric categorical categorical_logit ordered_logistic neg_binomial neg_binomial_2 neg_binomial_2_log poisson poisson_log multinomial normal exp_mod_normal skew_normal student_t cauchy double_exponential logistic gumbel lognormal chi_square inv_chi_square scaled_inv_chi_square exponential inv_gamma weibull frechet rayleigh wiener pareto pareto_type_2 von_mises uniform multi_normal multi_normal_prec multi_normal_cholesky multi_gp multi_gp_cholesky multi_student_t gaussian_dlm_obs dirichlet lkj_corr lkj_corr_cholesky wishart inv_wishart","selector-tag":"int real vector simplex unit_vector ordered positive_ordered row_vector matrix cholesky_factor_corr cholesky_factor_cov corr_matrix cov_matrix",title:"functions model data parameters quantities transformed generated",literal:"true false"},r:0},{cN:"number",b:"0[xX][0-9a-fA-F]+[Li]?\\b",r:0},{cN:"number",b:"0[xX][0-9a-fA-F]+[Li]?\\b",r:0},{cN:"number",b:"\\d+(?:[eE][+\\-]?\\d*)?L\\b",r:0},{cN:"number",b:"\\d+\\.(?!\\d)(?:i\\b)?",r:0},{cN:"number",b:"\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",r:0},{cN:"number",b:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",r:0}]}})),e.registerLanguage("stata",(function(e){return{aliases:["do","ado"],cI:!0,k:"if else in foreach for forv forva forval forvalu forvalue forvalues by bys bysort xi quietly qui capture about ac ac_7 acprplot acprplot_7 adjust ado adopath adoupdate alpha ameans an ano anov anova anova_estat anova_terms anovadef aorder ap app appe appen append arch arch_dr arch_estat arch_p archlm areg areg_p args arima arima_dr arima_estat arima_p as asmprobit asmprobit_estat asmprobit_lf asmprobit_mfx__dlg asmprobit_p ass asse asser assert avplot avplot_7 avplots avplots_7 bcskew0 bgodfrey binreg bip0_lf biplot bipp_lf bipr_lf bipr_p biprobit bitest bitesti bitowt blogit bmemsize boot bootsamp bootstrap bootstrap_8 boxco_l boxco_p boxcox boxcox_6 boxcox_p bprobit br break brier bro brow brows browse brr brrstat bs bs_7 bsampl_w bsample bsample_7 bsqreg bstat bstat_7 bstat_8 bstrap bstrap_7 ca ca_estat ca_p cabiplot camat canon canon_8 canon_8_p canon_estat canon_p cap caprojection capt captu captur capture cat cc cchart cchart_7 cci cd censobs_table centile cf char chdir checkdlgfiles checkestimationsample checkhlpfiles checksum chelp ci cii cl class classutil clear cli clis clist clo clog clog_lf clog_p clogi clogi_sw clogit clogit_lf clogit_p clogitp clogl_sw cloglog clonevar clslistarray cluster cluster_measures cluster_stop cluster_tree cluster_tree_8 clustermat cmdlog cnr cnre cnreg cnreg_p cnreg_sw cnsreg codebook collaps4 collapse colormult_nb colormult_nw compare compress conf confi confir confirm conren cons const constr constra constrai constrain constraint continue contract copy copyright copysource cor corc corr corr2data corr_anti corr_kmo corr_smc corre correl correla correlat correlate corrgram cou coun count cox cox_p cox_sw coxbase coxhaz coxvar cprplot cprplot_7 crc cret cretu cretur creturn cross cs cscript cscript_log csi ct ct_is ctset ctst_5 ctst_st cttost cumsp cumsp_7 cumul cusum cusum_7 cutil d|0 datasig datasign datasigna datasignat datasignatu datasignatur datasignature datetof db dbeta de dec deco decod decode deff des desc descr descri describ describe destring dfbeta dfgls dfuller di di_g dir dirstats dis discard disp disp_res disp_s displ displa display distinct do doe doed doedi doedit dotplot dotplot_7 dprobit drawnorm drop ds ds_util dstdize duplicates durbina dwstat dydx e|0 ed edi edit egen eivreg emdef en enc enco encod encode eq erase ereg ereg_lf ereg_p ereg_sw ereghet ereghet_glf ereghet_glf_sh ereghet_gp ereghet_ilf ereghet_ilf_sh ereghet_ip eret eretu eretur ereturn err erro error est est_cfexist est_cfname est_clickable est_expand est_hold est_table est_unhold est_unholdok estat estat_default estat_summ estat_vce_only esti estimates etodow etof etomdy ex exi exit expand expandcl fac fact facto factor factor_estat factor_p factor_pca_rotated factor_rotate factormat fcast fcast_compute fcast_graph fdades fdadesc fdadescr fdadescri fdadescrib fdadescribe fdasav fdasave fdause fh_st file open file read file close file filefilter fillin find_hlp_file findfile findit findit_7 fit fl fli flis flist for5_0 form forma format fpredict frac_154 frac_adj frac_chk frac_cox frac_ddp frac_dis frac_dv frac_in frac_mun frac_pp frac_pq frac_pv frac_wgt frac_xo fracgen fracplot fracplot_7 fracpoly fracpred fron_ex fron_hn fron_p fron_tn fron_tn2 frontier ftodate ftoe ftomdy ftowdate g|0 gamhet_glf gamhet_gp gamhet_ilf gamhet_ip gamma gamma_d2 gamma_p gamma_sw gammahet gdi_hexagon gdi_spokes ge gen gene gener genera generat generate genrank genstd genvmean gettoken gl gladder gladder_7 glim_l01 glim_l02 glim_l03 glim_l04 glim_l05 glim_l06 glim_l07 glim_l08 glim_l09 glim_l10 glim_l11 glim_l12 glim_lf glim_mu glim_nw1 glim_nw2 glim_nw3 glim_p glim_v1 glim_v2 glim_v3 glim_v4 glim_v5 glim_v6 glim_v7 glm glm_6 glm_p glm_sw glmpred glo glob globa global glogit glogit_8 glogit_p gmeans gnbre_lf gnbreg gnbreg_5 gnbreg_p gomp_lf gompe_sw gomper_p gompertz gompertzhet gomphet_glf gomphet_glf_sh gomphet_gp gomphet_ilf gomphet_ilf_sh gomphet_ip gphdot gphpen gphprint gprefs gprobi_p gprobit gprobit_8 gr gr7 gr_copy gr_current gr_db gr_describe gr_dir gr_draw gr_draw_replay gr_drop gr_edit gr_editviewopts gr_example gr_example2 gr_export gr_print gr_qscheme gr_query gr_read gr_rename gr_replay gr_save gr_set gr_setscheme gr_table gr_undo gr_use graph graph7 grebar greigen greigen_7 greigen_8 grmeanby grmeanby_7 gs_fileinfo gs_filetype gs_graphinfo gs_stat gsort gwood h|0 hadimvo hareg hausman haver he heck_d2 heckma_p heckman heckp_lf heckpr_p heckprob hel help hereg hetpr_lf hetpr_p hetprob hettest hexdump hilite hist hist_7 histogram hlogit hlu hmeans hotel hotelling hprobit hreg hsearch icd9 icd9_ff icd9p iis impute imtest inbase include inf infi infil infile infix inp inpu input ins insheet insp inspe inspec inspect integ inten intreg intreg_7 intreg_p intrg2_ll intrg_ll intrg_ll2 ipolate iqreg ir irf irf_create irfm iri is_svy is_svysum isid istdize ivprob_1_lf ivprob_lf ivprobit ivprobit_p ivreg ivreg_footnote ivtob_1_lf ivtob_lf ivtobit ivtobit_p jackknife jacknife jknife jknife_6 jknife_8 jkstat joinby kalarma1 kap kap_3 kapmeier kappa kapwgt kdensity kdensity_7 keep ksm ksmirnov ktau kwallis l|0 la lab labe label labelbook ladder levels levelsof leverage lfit lfit_p li lincom line linktest lis list lloghet_glf lloghet_glf_sh lloghet_gp lloghet_ilf lloghet_ilf_sh lloghet_ip llogi_sw llogis_p llogist llogistic llogistichet lnorm_lf lnorm_sw lnorma_p lnormal lnormalhet lnormhet_glf lnormhet_glf_sh lnormhet_gp lnormhet_ilf lnormhet_ilf_sh lnormhet_ip lnskew0 loadingplot loc loca local log logi logis_lf logistic logistic_p logit logit_estat logit_p loglogs logrank loneway lookfor lookup lowess lowess_7 lpredict lrecomp lroc lroc_7 lrtest ls lsens lsens_7 lsens_x lstat ltable ltable_7 ltriang lv lvr2plot lvr2plot_7 m|0 ma mac macr macro makecns man manova manova_estat manova_p manovatest mantel mark markin markout marksample mat mat_capp mat_order mat_put_rr mat_rapp mata mata_clear mata_describe mata_drop mata_matdescribe mata_matsave mata_matuse mata_memory mata_mlib mata_mosave mata_rename mata_which matalabel matcproc matlist matname matr matri matrix matrix_input__dlg matstrik mcc mcci md0_ md1_ md1debug_ md2_ md2debug_ mds mds_estat mds_p mdsconfig mdslong mdsmat mdsshepard mdytoe mdytof me_derd mean means median memory memsize meqparse mer merg merge mfp mfx mhelp mhodds minbound mixed_ll mixed_ll_reparm mkassert mkdir mkmat mkspline ml ml_5 ml_adjs ml_bhhhs ml_c_d ml_check ml_clear ml_cnt ml_debug ml_defd ml_e0 ml_e0_bfgs ml_e0_cycle ml_e0_dfp ml_e0i ml_e1 ml_e1_bfgs ml_e1_bhhh ml_e1_cycle ml_e1_dfp ml_e2 ml_e2_cycle ml_ebfg0 ml_ebfr0 ml_ebfr1 ml_ebh0q ml_ebhh0 ml_ebhr0 ml_ebr0i ml_ecr0i ml_edfp0 ml_edfr0 ml_edfr1 ml_edr0i ml_eds ml_eer0i ml_egr0i ml_elf ml_elf_bfgs ml_elf_bhhh ml_elf_cycle ml_elf_dfp ml_elfi ml_elfs ml_enr0i ml_enrr0 ml_erdu0 ml_erdu0_bfgs ml_erdu0_bhhh ml_erdu0_bhhhq ml_erdu0_cycle ml_erdu0_dfp ml_erdu0_nrbfgs ml_exde ml_footnote ml_geqnr ml_grad0 ml_graph ml_hbhhh ml_hd0 ml_hold ml_init ml_inv ml_log ml_max ml_mlout ml_mlout_8 ml_model ml_nb0 ml_opt ml_p ml_plot ml_query ml_rdgrd ml_repor ml_s_e ml_score ml_searc ml_technique ml_unhold mleval mlf_ mlmatbysum mlmatsum mlog mlogi mlogit mlogit_footnote mlogit_p mlopts mlsum mlvecsum mnl0_ mor more mov move mprobit mprobit_lf mprobit_p mrdu0_ mrdu1_ mvdecode mvencode mvreg mvreg_estat n|0 nbreg nbreg_al nbreg_lf nbreg_p nbreg_sw nestreg net newey newey_7 newey_p news nl nl_7 nl_9 nl_9_p nl_p nl_p_7 nlcom nlcom_p nlexp2 nlexp2_7 nlexp2a nlexp2a_7 nlexp3 nlexp3_7 nlgom3 nlgom3_7 nlgom4 nlgom4_7 nlinit nllog3 nllog3_7 nllog4 nllog4_7 nlog_rd nlogit nlogit_p nlogitgen nlogittree nlpred no nobreak noi nois noisi noisil noisily note notes notes_dlg nptrend numlabel numlist odbc old_ver olo olog ologi ologi_sw ologit ologit_p ologitp on one onew onewa oneway op_colnm op_comp op_diff op_inv op_str opr opro oprob oprob_sw oprobi oprobi_p oprobit oprobitp opts_exclusive order orthog orthpoly ou out outf outfi outfil outfile outs outsh outshe outshee outsheet ovtest pac pac_7 palette parse parse_dissim pause pca pca_8 pca_display pca_estat pca_p pca_rotate pcamat pchart pchart_7 pchi pchi_7 pcorr pctile pentium pergram pergram_7 permute permute_8 personal peto_st pkcollapse pkcross pkequiv pkexamine pkexamine_7 pkshape pksumm pksumm_7 pl plo plot plugin pnorm pnorm_7 poisgof poiss_lf poiss_sw poisso_p poisson poisson_estat post postclose postfile postutil pperron pr prais prais_e prais_e2 prais_p predict predictnl preserve print pro prob probi probit probit_estat probit_p proc_time procoverlay procrustes procrustes_estat procrustes_p profiler prog progr progra program prop proportion prtest prtesti pwcorr pwd q\\s qby qbys qchi qchi_7 qladder qladder_7 qnorm qnorm_7 qqplot qqplot_7 qreg qreg_c qreg_p qreg_sw qu quadchk quantile quantile_7 que quer query range ranksum ratio rchart rchart_7 rcof recast reclink recode reg reg3 reg3_p regdw regr regre regre_p2 regres regres_p regress regress_estat regriv_p remap ren rena renam rename renpfix repeat replace report reshape restore ret retu retur return rm rmdir robvar roccomp roccomp_7 roccomp_8 rocf_lf rocfit rocfit_8 rocgold rocplot rocplot_7 roctab roctab_7 rolling rologit rologit_p rot rota rotat rotate rotatemat rreg rreg_p ru run runtest rvfplot rvfplot_7 rvpplot rvpplot_7 sa safesum sample sampsi sav save savedresults saveold sc sca scal scala scalar scatter scm_mine sco scob_lf scob_p scobi_sw scobit scor score scoreplot scoreplot_help scree screeplot screeplot_help sdtest sdtesti se search separate separate serrbar serrbar_7 serset set set_defaults sfrancia sh she shel shell shewhart shewhart_7 signestimationsample signrank signtest simul simul_7 simulate simulate_8 sktest sleep slogit slogit_d2 slogit_p smooth snapspan so sor sort spearman spikeplot spikeplot_7 spikeplt spline_x split sqreg sqreg_p sret sretu sretur sreturn ssc st st_ct st_hc st_hcd st_hcd_sh st_is st_issys st_note st_promo st_set st_show st_smpl st_subid stack statsby statsby_8 stbase stci stci_7 stcox stcox_estat stcox_fr stcox_fr_ll stcox_p stcox_sw stcoxkm stcoxkm_7 stcstat stcurv stcurve stcurve_7 stdes stem stepwise stereg stfill stgen stir stjoin stmc stmh stphplot stphplot_7 stphtest stphtest_7 stptime strate strate_7 streg streg_sw streset sts sts_7 stset stsplit stsum sttocc sttoct stvary stweib su suest suest_8 sum summ summa summar summari summariz summarize sunflower sureg survcurv survsum svar svar_p svmat svy svy_disp svy_dreg svy_est svy_est_7 svy_estat svy_get svy_gnbreg_p svy_head svy_header svy_heckman_p svy_heckprob_p svy_intreg_p svy_ivreg_p svy_logistic_p svy_logit_p svy_mlogit_p svy_nbreg_p svy_ologit_p svy_oprobit_p svy_poisson_p svy_probit_p svy_regress_p svy_sub svy_sub_7 svy_x svy_x_7 svy_x_p svydes svydes_8 svygen svygnbreg svyheckman svyheckprob svyintreg svyintreg_7 svyintrg svyivreg svylc svylog_p svylogit svymarkout svymarkout_8 svymean svymlog svymlogit svynbreg svyolog svyologit svyoprob svyoprobit svyopts svypois svypois_7 svypoisson svyprobit svyprobt svyprop svyprop_7 svyratio svyreg svyreg_p svyregress svyset svyset_7 svyset_8 svytab svytab_7 svytest svytotal sw sw_8 swcnreg swcox swereg swilk swlogis swlogit swologit swoprbt swpois swprobit swqreg swtobit swweib symmetry symmi symplot symplot_7 syntax sysdescribe sysdir sysuse szroeter ta tab tab1 tab2 tab_or tabd tabdi tabdis tabdisp tabi table tabodds tabodds_7 tabstat tabu tabul tabula tabulat tabulate te tempfile tempname tempvar tes test testnl testparm teststd tetrachoric time_it timer tis tob tobi tobit tobit_p tobit_sw token tokeni tokeniz tokenize tostring total translate translator transmap treat_ll treatr_p treatreg trim trnb_cons trnb_mean trpoiss_d2 trunc_ll truncr_p truncreg tsappend tset tsfill tsline tsline_ex tsreport tsrevar tsrline tsset tssmooth tsunab ttest ttesti tut_chk tut_wait tutorial tw tware_st two twoway twoway__fpfit_serset twoway__function_gen twoway__histogram_gen twoway__ipoint_serset twoway__ipoints_serset twoway__kdensity_gen twoway__lfit_serset twoway__normgen_gen twoway__pci_serset twoway__qfit_serset twoway__scatteri_serset twoway__sunflower_gen twoway_ksm_serset ty typ type typeof u|0 unab unabbrev unabcmd update us use uselabel var var_mkcompanion var_p varbasic varfcast vargranger varirf varirf_add varirf_cgraph varirf_create varirf_ctable varirf_describe varirf_dir varirf_drop varirf_erase varirf_graph varirf_ograph varirf_rename varirf_set varirf_table varlist varlmar varnorm varsoc varstable varstable_w varstable_w2 varwle vce vec vec_fevd vec_mkphi vec_p vec_p_w vecirf_create veclmar veclmar_w vecnorm vecnorm_w vecrank vecstable verinst vers versi versio version view viewsource vif vwls wdatetof webdescribe webseek webuse weib1_lf weib2_lf weib_lf weib_lf0 weibhet_glf weibhet_glf_sh weibhet_glfa weibhet_glfa_sh weibhet_gp weibhet_ilf weibhet_ilf_sh weibhet_ilfa weibhet_ilfa_sh weibhet_ip weibu_sw weibul_p weibull weibull_c weibull_s weibullhet wh whelp whi which whil while wilc_st wilcoxon win wind windo window winexec wntestb wntestb_7 wntestq xchart xchart_7 xcorr xcorr_7 xi xi_6 xmlsav xmlsave xmluse xpose xsh xshe xshel xshell xt_iis xt_tis xtab_p xtabond xtbin_p xtclog xtcloglog xtcloglog_8 xtcloglog_d2 xtcloglog_pa_p xtcloglog_re_p xtcnt_p xtcorr xtdata xtdes xtfront_p xtfrontier xtgee xtgee_elink xtgee_estat xtgee_makeivar xtgee_p xtgee_plink xtgls xtgls_p xthaus xthausman xtht_p xthtaylor xtile xtint_p xtintreg xtintreg_8 xtintreg_d2 xtintreg_p xtivp_1 xtivp_2 xtivreg xtline xtline_ex xtlogit xtlogit_8 xtlogit_d2 xtlogit_fe_p xtlogit_pa_p xtlogit_re_p xtmixed xtmixed_estat xtmixed_p xtnb_fe xtnb_lf xtnbreg xtnbreg_pa_p xtnbreg_refe_p xtpcse xtpcse_p xtpois xtpoisson xtpoisson_d2 xtpoisson_pa_p xtpoisson_refe_p xtpred xtprobit xtprobit_8 xtprobit_d2 xtprobit_re_p xtps_fe xtps_lf xtps_ren xtps_ren_8 xtrar_p xtrc xtrc_p xtrchh xtrefe_p xtreg xtreg_be xtreg_fe xtreg_ml xtreg_pa_p xtreg_re xtregar xtrere_p xtset xtsf_ll xtsf_llti xtsum xttab xttest0 xttobit xttobit_8 xttobit_p xttrans yx yxview__barlike_draw yxview_area_draw yxview_bar_draw yxview_dot_draw yxview_dropline_draw yxview_function_draw yxview_iarrow_draw yxview_ilabels_draw yxview_normal_draw yxview_pcarrow_draw yxview_pcbarrow_draw yxview_pccapsym_draw yxview_pcscatter_draw yxview_pcspike_draw yxview_rarea_draw yxview_rbar_draw yxview_rbarm_draw yxview_rcap_draw yxview_rcapsym_draw yxview_rconnected_draw yxview_rline_draw yxview_rscatter_draw yxview_rspike_draw yxview_spike_draw yxview_sunflower_draw zap_s zinb zinb_llf zinb_plf zip zip_llf zip_p zip_plf zt_ct_5 zt_hc_5 zt_hcd_5 zt_is_5 zt_iss_5 zt_sho_5 zt_smp_5 ztbase_5 ztcox_5 ztdes_5 ztereg_5 ztfill_5 ztgen_5 ztir_5 ztjoin_5 ztnb ztnb_p ztp ztp_p zts_5 ztset_5 ztspli_5 ztsum_5 zttoct_5 ztvary_5 ztweib_5",c:[{cN:"symbol",b:/`[a-zA-Z0-9_]+'/},{cN:"variable",b:/\$\{?[a-zA-Z0-9_]+\}?/},{cN:"string",v:[{b:'`"[^\r\n]*?"\''},{b:'"[^\r\n"]*"'}]},{cN:"built_in",v:[{b:"\\b(abs|acos|asin|atan|atan2|atanh|ceil|cloglog|comb|cos|digamma|exp|floor|invcloglog|invlogit|ln|lnfact|lnfactorial|lngamma|log|log10|max|min|mod|reldif|round|sign|sin|sqrt|sum|tan|tanh|trigamma|trunc|betaden|Binomial|binorm|binormal|chi2|chi2tail|dgammapda|dgammapdada|dgammapdadx|dgammapdx|dgammapdxdx|F|Fden|Ftail|gammaden|gammap|ibeta|invbinomial|invchi2|invchi2tail|invF|invFtail|invgammap|invibeta|invnchi2|invnFtail|invnibeta|invnorm|invnormal|invttail|nbetaden|nchi2|nFden|nFtail|nibeta|norm|normal|normalden|normd|npnchi2|tden|ttail|uniform|abbrev|char|index|indexnot|length|lower|ltrim|match|plural|proper|real|regexm|regexr|regexs|reverse|rtrim|string|strlen|strlower|strltrim|strmatch|strofreal|strpos|strproper|strreverse|strrtrim|strtrim|strupper|subinstr|subinword|substr|trim|upper|word|wordcount|_caller|autocode|byteorder|chop|clip|cond|e|epsdouble|epsfloat|group|inlist|inrange|irecode|matrix|maxbyte|maxdouble|maxfloat|maxint|maxlong|mi|minbyte|mindouble|minfloat|minint|minlong|missing|r|recode|replay|return|s|scalar|d|date|day|dow|doy|halfyear|mdy|month|quarter|week|year|d|daily|dofd|dofh|dofm|dofq|dofw|dofy|h|halfyearly|hofd|m|mofd|monthly|q|qofd|quarterly|tin|twithin|w|weekly|wofd|y|yearly|yh|ym|yofd|yq|yw|cholesky|colnumb|colsof|corr|det|diag|diag0cnt|el|get|hadamard|I|inv|invsym|issym|issymmetric|J|matmissing|matuniform|mreldif|nullmat|rownumb|rowsof|sweep|syminv|trace|vec|vecdiag)(?=\\(|$)"}]},e.C("^[ \t]*\\*.*$",!1),e.CLCM,e.CBCM]}})),e.registerLanguage("step21",(function(e){var t="[A-Z_][A-Z0-9_.]*",r={keyword:"HEADER ENDSEC DATA"},a={cN:"meta",b:"ISO-10303-21;",r:10},i={cN:"meta",b:"END-ISO-10303-21;",r:10};return{aliases:["p21","step","stp"],cI:!0,l:t,k:r,c:[a,i,e.CLCM,e.CBCM,e.C("/\\*\\*!","\\*/"),e.CNM,e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null}),{cN:"string",b:"'",e:"'"},{cN:"symbol",v:[{b:"#",e:"\\d+",i:"\\W"}]}]}})),e.registerLanguage("stylus",(function(e){var t={cN:"variable",b:"\\$"+e.IR},r={cN:"number",b:"#([a-fA-F0-9]{6}|[a-fA-F0-9]{3})"},a=["charset","css","debug","extend","font-face","for","import","include","media","mixin","page","warn","while"],i=["after","before","first-letter","first-line","active","first-child","focus","hover","lang","link","visited"],n=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","mark","menu","nav","object","ol","p","q","quote","samp","section","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video"],o="[\\.\\s\\n\\[\\:,]",s=["align-content","align-items","align-self","animation","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-timing-function","auto","backface-visibility","background","background-attachment","background-clip","background-color","background-image","background-origin","background-position","background-repeat","background-size","border","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-decoration-break","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","clear","clip","clip-path","color","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","content","counter-increment","counter-reset","cursor","direction","display","empty-cells","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","font","font-family","font-feature-settings","font-kerning","font-language-override","font-size","font-size-adjust","font-stretch","font-style","font-variant","font-variant-ligatures","font-weight","height","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","inherit","initial","justify-content","left","letter-spacing","line-height","list-style","list-style-image","list-style-position","list-style-type","margin","margin-bottom","margin-left","margin-right","margin-top","marks","mask","max-height","max-width","min-height","min-width","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-wrap","overflow-x","overflow-y","padding","padding-bottom","padding-left","padding-right","padding-top","page-break-after","page-break-before","page-break-inside","perspective","perspective-origin","pointer-events","position","quotes","resize","right","tab-size","table-layout","text-align","text-align-last","text-decoration","text-decoration-color","text-decoration-line","text-decoration-style","text-indent","text-overflow","text-rendering","text-shadow","text-transform","text-underline-position","top","transform","transform-origin","transform-style","transition","transition-delay","transition-duration","transition-property","transition-timing-function","unicode-bidi","vertical-align","visibility","white-space","widows","width","word-break","word-spacing","word-wrap","z-index"],l=["\\?","(\\bReturn\\b)","(\\bEnd\\b)","(\\bend\\b)","(\\bdef\\b)",";","#\\s","\\*\\s","===\\s","\\|","%"];return{aliases:["styl"],cI:!1,k:"if else for in",i:"("+l.join("|")+")",c:[e.QSM,e.ASM,e.CLCM,e.CBCM,r,{b:"\\.[a-zA-Z][a-zA-Z0-9_-]*"+o,rB:!0,c:[{cN:"selector-class",b:"\\.[a-zA-Z][a-zA-Z0-9_-]*"}]},{b:"\\#[a-zA-Z][a-zA-Z0-9_-]*"+o,rB:!0,c:[{cN:"selector-id",b:"\\#[a-zA-Z][a-zA-Z0-9_-]*"}]},{b:"\\b("+n.join("|")+")"+o,rB:!0,c:[{cN:"selector-tag",b:"\\b[a-zA-Z][a-zA-Z0-9_-]*"}]},{b:"&?:?:\\b("+i.join("|")+")"+o},{b:"@("+a.join("|")+")\\b"},t,e.CSSNM,e.NM,{cN:"function",b:"^[a-zA-Z][a-zA-Z0-9_-]*\\(.*\\)",i:"[\\n]",rB:!0,c:[{cN:"title",b:"\\b[a-zA-Z][a-zA-Z0-9_-]*"},{cN:"params",b:/\(/,e:/\)/,c:[r,t,e.ASM,e.CSSNM,e.NM,e.QSM]}]},{cN:"attribute",b:"\\b("+s.reverse().join("|")+")\\b",starts:{e:/;|$/,c:[r,t,e.ASM,e.QSM,e.CSSNM,e.NM,e.CBCM],i:/\./,r:0}}]}})),e.registerLanguage("subunit",(function(e){var t={cN:"string",b:"\\[\n(multipart)?",e:"\\]\n"},r={cN:"string",b:"\\d{4}-\\d{2}-\\d{2}(\\s+)\\d{2}:\\d{2}:\\d{2}.\\d+Z"},a={cN:"string",b:"(\\+|-)\\d+"},i={cN:"keyword",r:10,v:[{b:"^(test|testing|success|successful|failure|error|skip|xfail|uxsuccess)(:?)\\s+(test)?"},{b:"^progress(:?)(\\s+)?(pop|push)?"},{b:"^tags:"},{b:"^time:"}]};return{cI:!0,c:[t,r,a,i]}})),e.registerLanguage("swift",(function(e){var t={keyword:"__COLUMN__ __FILE__ __FUNCTION__ __LINE__ as as! as? associativity break case catch class continue convenience default defer deinit didSet do dynamic dynamicType else enum extension fallthrough false fileprivate final for func get guard if import in indirect infix init inout internal is lazy left let mutating nil none nonmutating open operator optional override postfix precedence prefix private protocol Protocol public repeat required rethrows return right self Self set static struct subscript super switch throw throws true try try! try? Type typealias unowned var weak where while willSet",literal:"true false nil",built_in:"abs advance alignof alignofValue anyGenerator assert assertionFailure bridgeFromObjectiveC bridgeFromObjectiveCUnconditional bridgeToObjectiveC bridgeToObjectiveCUnconditional c contains count countElements countLeadingZeros debugPrint debugPrintln distance dropFirst dropLast dump encodeBitsAsWords enumerate equal fatalError filter find getBridgedObjectiveCType getVaList indices insertionSort isBridgedToObjectiveC isBridgedVerbatimToObjectiveC isUniquelyReferenced isUniquelyReferencedNonObjC join lazy lexicographicalCompare map max maxElement min minElement numericCast overlaps partition posix precondition preconditionFailure print println quickSort readLine reduce reflect reinterpretCast reverse roundUpToAlignment sizeof sizeofValue sort split startsWith stride strideof strideofValue swap toString transcode underestimateCount unsafeAddressOf unsafeBitCast unsafeDowncast unsafeUnwrap unsafeReflect withExtendedLifetime withObjectAtPlusZero withUnsafePointer withUnsafePointerToObject withUnsafeMutablePointer withUnsafeMutablePointers withUnsafePointer withUnsafePointers withVaList zip"},r={cN:"type",b:"\\b[A-Z][\\wÀ-ʸ']*",r:0},a=e.C("/\\*","\\*/",{c:["self"]}),i={cN:"subst",b:/\\\(/,e:"\\)",k:t,c:[]},n={cN:"number",b:"\\b([\\d_]+(\\.[\\deE_]+)?|0x[a-fA-F0-9_]+(\\.[a-fA-F0-9p_]+)?|0b[01_]+|0o[0-7_]+)\\b",r:0},o=e.inherit(e.QSM,{c:[i,e.BE]});return i.c=[n],{k:t,c:[o,e.CLCM,a,r,n,{cN:"function",bK:"func",e:"{",eE:!0,c:[e.inherit(e.TM,{b:/[A-Za-z$_][0-9A-Za-z$_]*/}),{b://},{cN:"params",b:/\(/,e:/\)/,endsParent:!0,k:t,c:["self",n,o,e.CBCM,{b:":"}],i:/["']/}],i:/\[|%/},{cN:"class",bK:"struct protocol class extension enum",k:t,e:"\\{",eE:!0,c:[e.inherit(e.TM,{b:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/})]},{cN:"meta",b:"(@warn_unused_result|@exported|@lazy|@noescape|@NSCopying|@NSManaged|@objc|@convention|@required|@noreturn|@IBAction|@IBDesignable|@IBInspectable|@IBOutlet|@infix|@prefix|@postfix|@autoclosure|@testable|@available|@nonobjc|@NSApplicationMain|@UIApplicationMain)"},{bK:"import",e:/$/,c:[e.CLCM,a]}]}})),e.registerLanguage("taggerscript",(function(e){var t={cN:"comment",b:/\$noop\(/,e:/\)/,c:[{b:/\(/,e:/\)/,c:["self",{b:/\\./}]}],r:10},r={cN:"keyword",b:/\$(?!noop)[a-zA-Z][_a-zA-Z0-9]*/,e:/\(/,eE:!0},a={cN:"variable",b:/%[_a-zA-Z0-9:]*/,e:"%"},i={cN:"symbol",b:/\\./};return{c:[t,r,a,i]}})),e.registerLanguage("yaml",(function(e){var t="true false yes no null",r="^[ \\-]*",a="[a-zA-Z_][\\w\\-]*",i={cN:"attr",v:[{b:r+a+":"},{b:r+'"'+a+'":'},{b:r+"'"+a+"':"}]},n={cN:"template-variable",v:[{b:"{{",e:"}}"},{b:"%{",e:"}"}]},o={cN:"string",r:0,v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/\S+/}],c:[e.BE,n]};return{cI:!0,aliases:["yml","YAML","yaml"],c:[i,{cN:"meta",b:"^---s*$",r:10},{cN:"string",b:"[\\|>] *$",rE:!0,c:o.c,e:i.v[0].b},{b:"<%[%=-]?",e:"[%-]?%>",sL:"ruby",eB:!0,eE:!0,r:0},{cN:"type",b:"!!"+e.UIR},{cN:"meta",b:"&"+e.UIR+"$"},{cN:"meta",b:"\\*"+e.UIR+"$"},{cN:"bullet",b:"^ *-",r:0},e.HCM,{bK:t,k:{literal:t}},e.CNM,o]}})),e.registerLanguage("tap",(function(e){return{cI:!0,c:[e.HCM,{cN:"meta",v:[{b:"^TAP version (\\d+)$"},{b:"^1\\.\\.(\\d+)$"}]},{b:"(s+)?---$",e:"\\.\\.\\.$",sL:"yaml",r:0},{cN:"number",b:" (\\d+) "},{cN:"symbol",v:[{b:"^ok"},{b:"^not ok"}]}]}})),e.registerLanguage("tcl",(function(e){return{aliases:["tk"],k:"after append apply array auto_execok auto_import auto_load auto_mkindex auto_mkindex_old auto_qualify auto_reset bgerror binary break catch cd chan clock close concat continue dde dict encoding eof error eval exec exit expr fblocked fconfigure fcopy file fileevent filename flush for foreach format gets glob global history http if incr info interp join lappend|10 lassign|10 lindex|10 linsert|10 list llength|10 load lrange|10 lrepeat|10 lreplace|10 lreverse|10 lsearch|10 lset|10 lsort|10 mathfunc mathop memory msgcat namespace open package parray pid pkg::create pkg_mkIndex platform platform::shell proc puts pwd read refchan regexp registry regsub|10 rename return safe scan seek set socket source split string subst switch tcl_endOfWord tcl_findLibrary tcl_startOfNextWord tcl_startOfPreviousWord tcl_wordBreakAfter tcl_wordBreakBefore tcltest tclvars tell time tm trace unknown unload unset update uplevel upvar variable vwait while",c:[e.C(";[ \\t]*#","$"),e.C("^[ \\t]*#","$"),{bK:"proc",e:"[\\{]",eE:!0,c:[{cN:"title",b:"[ \\t\\n\\r]+(::)?[a-zA-Z_]((::)?[a-zA-Z0-9_])*",e:"[ \\t\\n\\r]",eW:!0,eE:!0}]},{eE:!0,v:[{b:"\\$(\\{)?(::)?[a-zA-Z_]((::)?[a-zA-Z0-9_])*\\(([a-zA-Z0-9_])*\\)",e:"[^a-zA-Z0-9_\\}\\$]"},{b:"\\$(\\{)?(::)?[a-zA-Z_]((::)?[a-zA-Z0-9_])*",e:"(\\))?[^a-zA-Z0-9_\\}\\$]"}]},{cN:"string",c:[e.BE],v:[e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null})]},{cN:"number",v:[e.BNM,e.CNM]}]}})),e.registerLanguage("tex",(function(e){var t={cN:"tag",b:/\\/,r:0,c:[{cN:"name",v:[{b:/[a-zA-Zа-ÑÐ-Ñ]+[*]?/},{b:/[^a-zA-Zа-ÑÐ-Ñ0-9]/}],starts:{eW:!0,r:0,c:[{cN:"string",v:[{b:/\[/,e:/\]/},{b:/\{/,e:/\}/}]},{b:/\s*=\s*/,eW:!0,r:0,c:[{cN:"number",b:/-?\d*\.?\d+(pt|pc|mm|cm|in|dd|cc|ex|em)?/}]}]}}]};return{c:[t,{cN:"formula",c:[t],r:0,v:[{b:/\$\$/,e:/\$\$/},{b:/\$/,e:/\$/}]},e.C("%","$",{r:0})]}})),e.registerLanguage("thrift",(function(e){var t="bool byte i16 i32 i64 double string binary";return{k:{keyword:"namespace const typedef struct enum service exception void oneway set list map required optional",built_in:t,literal:"true false"},c:[e.QSM,e.NM,e.CLCM,e.CBCM,{cN:"class",bK:"struct enum service exception",e:/\{/,i:/\n/,c:[e.inherit(e.TM,{starts:{eW:!0,eE:!0}})]},{b:"\\b(set|list|map)\\s*<",e:">",k:t,c:["self"]}]}})),e.registerLanguage("tp",(function(e){var t={cN:"number",b:"[1-9][0-9]*",r:0},r={cN:"symbol",b:":[^\\]]+"},a={cN:"built_in",b:"(AR|P|PAYLOAD|PR|R|SR|RSR|LBL|VR|UALM|MESSAGE|UTOOL|UFRAME|TIMER| TIMER_OVERFLOW|JOINT_MAX_SPEED|RESUME_PROG|DIAG_REC)\\[",e:"\\]",c:["self",t,r]},i={cN:"built_in",b:"(AI|AO|DI|DO|F|RI|RO|UI|UO|GI|GO|SI|SO)\\[",e:"\\]",c:["self",t,e.QSM,r]};return{k:{keyword:"ABORT ACC ADJUST AND AP_LD BREAK CALL CNT COL CONDITION CONFIG DA DB DIV DETECT ELSE END ENDFOR ERR_NUM ERROR_PROG FINE FOR GP GUARD INC IF JMP LINEAR_MAX_SPEED LOCK MOD MONITOR OFFSET Offset OR OVERRIDE PAUSE PREG PTH RT_LD RUN SELECT SKIP Skip TA TB TO TOOL_OFFSET Tool_Offset UF UT UFRAME_NUM UTOOL_NUM UNLOCK WAIT X Y Z W P R STRLEN SUBSTR FINDSTR VOFFSET PROG ATTR MN POS",literal:"ON OFF max_speed LPOS JPOS ENABLE DISABLE START STOP RESET"},c:[a,i,{cN:"keyword",b:"/(PROG|ATTR|MN|POS|END)\\b"},{cN:"keyword",b:"(CALL|RUN|POINT_LOGIC|LBL)\\b"},{cN:"keyword",b:"\\b(ACC|CNT|Skip|Offset|PSPD|RT_LD|AP_LD|Tool_Offset)"},{cN:"number",b:"\\d+(sec|msec|mm/sec|cm/min|inch/min|deg/sec|mm|in|cm)?\\b",r:0},e.C("//","[;$]"),e.C("!","[;$]"),e.C("--eg:","$"),e.QSM,{cN:"string",b:"'",e:"'"},e.CNM,{cN:"variable",b:"\\$[A-Za-z0-9_]+"}]}})),e.registerLanguage("twig",(function(e){var t={cN:"params",b:"\\(",e:"\\)"},r="attribute block constant cycle date dump include max min parent random range source template_from_string",a={bK:r,k:{name:r},r:0,c:[t]},i={b:/\|[A-Za-z_]+:?/,k:"abs batch capitalize convert_encoding date date_modify default escape first format join json_encode keys last length lower merge nl2br number_format raw replace reverse round slice sort split striptags title trim upper url_encode",c:[a]},n="autoescape block do embed extends filter flush for if import include macro sandbox set spaceless use verbatim";return n=n+" "+n.split(" ").map((function(e){return"end"+e})).join(" "),{aliases:["craftcms"],cI:!0,sL:"xml",c:[e.C(/\{#/,/#}/),{cN:"template-tag",b:/\{%/,e:/%}/,c:[{cN:"name",b:/\w+/,k:n,starts:{eW:!0,c:[i,a],r:0}}]},{cN:"template-variable",b:/\{\{/,e:/}}/,c:["self",i,a]}]}})),e.registerLanguage("typescript",(function(e){var t={keyword:"in if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const class public private protected get set super static implements enum export import declare type namespace abstract as from extends async await",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document any number boolean string void Promise"};return{aliases:["ts"],k:t,c:[{cN:"meta",b:/^\s*['"]use strict['"]/},e.ASM,e.QSM,{cN:"string",b:"`",e:"`",c:[e.BE,{cN:"subst",b:"\\$\\{",e:"\\}"}]},e.CLCM,e.CBCM,{cN:"number",v:[{b:"\\b(0[bB][01]+)"},{b:"\\b(0[oO][0-7]+)"},{b:e.CNR}],r:0},{b:"("+e.RSR+"|\\b(case|return|throw)\\b)\\s*",k:"return throw case",c:[e.CLCM,e.CBCM,e.RM,{cN:"function",b:"(\\(.*?\\)|"+e.IR+")\\s*=>",rB:!0,e:"\\s*=>",c:[{cN:"params",v:[{b:e.IR},{b:/\(\s*\)/},{b:/\(/,e:/\)/,eB:!0,eE:!0,k:t,c:["self",e.CLCM,e.CBCM]}]}]}],r:0},{cN:"function",b:"function",e:/[\{;]/,eE:!0,k:t,c:["self",e.inherit(e.TM,{b:/[A-Za-z$_][0-9A-Za-z$_]*/}),{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:t,c:[e.CLCM,e.CBCM],i:/["'\(]/}],i:/%/,r:0},{bK:"constructor",e:/\{/,eE:!0,c:["self",{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:t,c:[e.CLCM,e.CBCM],i:/["'\(]/}]},{b:/module\./,k:{built_in:"module"},r:0},{bK:"module",e:/\{/,eE:!0},{bK:"interface",e:/\{/,eE:!0,k:"interface extends"},{b:/\$[(.]/},{b:"\\."+e.IR,r:0},{cN:"meta",b:"@[A-Za-z]+"}]}})),e.registerLanguage("vala",(function(e){return{k:{keyword:"char uchar unichar int uint long ulong short ushort int8 int16 int32 int64 uint8 uint16 uint32 uint64 float double bool struct enum string void weak unowned owned async signal static abstract interface override virtual delegate if while do for foreach else switch case break default return try catch public private protected internal using new this get set const stdout stdin stderr var",built_in:"DBus GLib CCode Gee Object Gtk Posix",literal:"false true null"},c:[{cN:"class",bK:"class interface namespace",e:"{",eE:!0,i:"[^,:\\n\\s\\.]",c:[e.UTM]},e.CLCM,e.CBCM,{cN:"string",b:'"""',e:'"""',r:5},e.ASM,e.QSM,e.CNM,{cN:"meta",b:"^#",e:"$",r:2}]}})),e.registerLanguage("vbnet",(function(e){return{aliases:["vb"],cI:!0,k:{keyword:"addhandler addressof alias and andalso aggregate ansi as assembly auto binary by byref byval call case catch class compare const continue custom declare default delegate dim distinct do each equals else elseif end enum erase error event exit explicit finally for friend from function get global goto group handles if implements imports in inherits interface into is isfalse isnot istrue join key let lib like loop me mid mod module mustinherit mustoverride mybase myclass namespace narrowing new next not notinheritable notoverridable of off on operator option optional or order orelse overloads overridable overrides paramarray partial preserve private property protected public raiseevent readonly redim rem removehandler resume return select set shadows shared skip static step stop structure strict sub synclock take text then throw to try unicode until using when where while widening with withevents writeonly xor",built_in:"boolean byte cbool cbyte cchar cdate cdec cdbl char cint clng cobj csbyte cshort csng cstr ctype date decimal directcast double gettype getxmlnamespace iif integer long object sbyte short single string trycast typeof uinteger ulong ushort",literal:"true false nothing"},i:"//|{|}|endif|gosub|variant|wend",c:[e.inherit(e.QSM,{c:[{b:'""'}]}),e.C("'","$",{rB:!0,c:[{cN:"doctag",b:"'''|\x3c!--|--\x3e",c:[e.PWM]},{cN:"doctag",b:"",c:[e.PWM]}]}),e.CNM,{cN:"meta",b:"#",e:"$",k:{"meta-keyword":"if else elseif end region externalsource"}}]}})),e.registerLanguage("vbscript",(function(e){return{aliases:["vbs"],cI:!0,k:{keyword:"call class const dim do loop erase execute executeglobal exit for each next function if then else on error option explicit new private property let get public randomize redim rem select case set stop sub while wend with end to elseif is or xor and not class_initialize class_terminate default preserve in me byval byref step resume goto",built_in:"lcase month vartype instrrev ubound setlocale getobject rgb getref string weekdayname rnd dateadd monthname now day minute isarray cbool round formatcurrency conversions csng timevalue second year space abs clng timeserial fixs len asc isempty maths dateserial atn timer isobject filter weekday datevalue ccur isdate instr datediff formatdatetime replace isnull right sgn array snumeric log cdbl hex chr lbound msgbox ucase getlocale cos cdate cbyte rtrim join hour oct typename trim strcomp int createobject loadpicture tan formatnumber mid scriptenginebuildversion scriptengine split scriptengineminorversion cint sin datepart ltrim sqr scriptenginemajorversion time derived eval date formatpercent exp inputbox left ascw chrw regexp server response request cstr err",literal:"true false null nothing empty"},i:"//",c:[e.inherit(e.QSM,{c:[{b:'""'}]}),e.C(/'/,/$/,{r:0}),e.CNM]}})),e.registerLanguage("vbscript-html",(function(e){return{sL:"xml",c:[{b:"<%",e:"%>",sL:"vbscript"}]}})),e.registerLanguage("verilog",(function(e){var t={keyword:"accept_on alias always always_comb always_ff always_latch and assert assign assume automatic before begin bind bins binsof bit break buf|0 bufif0 bufif1 byte case casex casez cell chandle checker class clocking cmos config const constraint context continue cover covergroup coverpoint cross deassign default defparam design disable dist do edge else end endcase endchecker endclass endclocking endconfig endfunction endgenerate endgroup endinterface endmodule endpackage endprimitive endprogram endproperty endspecify endsequence endtable endtask enum event eventually expect export extends extern final first_match for force foreach forever fork forkjoin function generate|5 genvar global highz0 highz1 if iff ifnone ignore_bins illegal_bins implements implies import incdir include initial inout input inside instance int integer interconnect interface intersect join join_any join_none large let liblist library local localparam logic longint macromodule matches medium modport module nand negedge nettype new nexttime nmos nor noshowcancelled not notif0 notif1 or output package packed parameter pmos posedge primitive priority program property protected pull0 pull1 pulldown pullup pulsestyle_ondetect pulsestyle_onevent pure rand randc randcase randsequence rcmos real realtime ref reg reject_on release repeat restrict return rnmos rpmos rtran rtranif0 rtranif1 s_always s_eventually s_nexttime s_until s_until_with scalared sequence shortint shortreal showcancelled signed small soft solve specify specparam static string strong strong0 strong1 struct super supply0 supply1 sync_accept_on sync_reject_on table tagged task this throughout time timeprecision timeunit tran tranif0 tranif1 tri tri0 tri1 triand trior trireg type typedef union unique unique0 unsigned until until_with untyped use uwire var vectored virtual void wait wait_order wand weak weak0 weak1 while wildcard wire with within wor xnor xor",literal:"null",built_in:"$finish $stop $exit $fatal $error $warning $info $realtime $time $printtimescale $bitstoreal $bitstoshortreal $itor $signed $cast $bits $stime $timeformat $realtobits $shortrealtobits $rtoi $unsigned $asserton $assertkill $assertpasson $assertfailon $assertnonvacuouson $assertoff $assertcontrol $assertpassoff $assertfailoff $assertvacuousoff $isunbounded $sampled $fell $changed $past_gclk $fell_gclk $changed_gclk $rising_gclk $steady_gclk $coverage_control $coverage_get $coverage_save $set_coverage_db_name $rose $stable $past $rose_gclk $stable_gclk $future_gclk $falling_gclk $changing_gclk $display $coverage_get_max $coverage_merge $get_coverage $load_coverage_db $typename $unpacked_dimensions $left $low $increment $clog2 $ln $log10 $exp $sqrt $pow $floor $ceil $sin $cos $tan $countbits $onehot $isunknown $fatal $warning $dimensions $right $high $size $asin $acos $atan $atan2 $hypot $sinh $cosh $tanh $asinh $acosh $atanh $countones $onehot0 $error $info $random $dist_chi_square $dist_erlang $dist_exponential $dist_normal $dist_poisson $dist_t $dist_uniform $q_initialize $q_remove $q_exam $async$and$array $async$nand$array $async$or$array $async$nor$array $sync$and$array $sync$nand$array $sync$or$array $sync$nor$array $q_add $q_full $psprintf $async$and$plane $async$nand$plane $async$or$plane $async$nor$plane $sync$and$plane $sync$nand$plane $sync$or$plane $sync$nor$plane $system $display $displayb $displayh $displayo $strobe $strobeb $strobeh $strobeo $write $readmemb $readmemh $writememh $value$plusargs $dumpvars $dumpon $dumplimit $dumpports $dumpportson $dumpportslimit $writeb $writeh $writeo $monitor $monitorb $monitorh $monitoro $writememb $dumpfile $dumpoff $dumpall $dumpflush $dumpportsoff $dumpportsall $dumpportsflush $fclose $fdisplay $fdisplayb $fdisplayh $fdisplayo $fstrobe $fstrobeb $fstrobeh $fstrobeo $swrite $swriteb $swriteh $swriteo $fscanf $fread $fseek $fflush $feof $fopen $fwrite $fwriteb $fwriteh $fwriteo $fmonitor $fmonitorb $fmonitorh $fmonitoro $sformat $sformatf $fgetc $ungetc $fgets $sscanf $rewind $ftell $ferror"};return{aliases:["v","sv","svh"],cI:!1,k:t,l:/[\w\$]+/,c:[e.CBCM,e.CLCM,e.QSM,{cN:"number",c:[e.BE],v:[{b:"\\b((\\d+'(b|h|o|d|B|H|O|D))[0-9xzXZa-fA-F_]+)"},{b:"\\B(('(b|h|o|d|B|H|O|D))[0-9xzXZa-fA-F_]+)"},{b:"\\b([0-9_])+",r:0}]},{cN:"variable",v:[{b:"#\\((?!parameter).+\\)"},{b:"\\.\\w+",r:0}]},{cN:"meta",b:"`",e:"$",k:{"meta-keyword":"define __FILE__ __LINE__ begin_keywords celldefine default_nettype define else elsif end_keywords endcelldefine endif ifdef ifndef include line nounconnected_drive pragma resetall timescale unconnected_drive undef undefineall"},r:0}]}})),e.registerLanguage("vhdl",(function(e){var t="\\d(_|\\d)*",r="[eE][-+]?"+t,a=t+"(\\."+t+")?("+r+")?",i="\\w+",n=t+"#"+i+"(\\."+i+")?#("+r+")?",o="\\b("+n+"|"+a+")";return{cI:!0,k:{keyword:"abs access after alias all and architecture array assert assume assume_guarantee attribute begin block body buffer bus case component configuration constant context cover disconnect downto default else elsif end entity exit fairness file for force function generate generic group guarded if impure in inertial inout is label library linkage literal loop map mod nand new next nor not null of on open or others out package port postponed procedure process property protected pure range record register reject release rem report restrict restrict_guarantee return rol ror select sequence severity shared signal sla sll sra srl strong subtype then to transport type unaffected units until use variable vmode vprop vunit wait when while with xnor xor",built_in:"boolean bit character integer time delay_length natural positive string bit_vector file_open_kind file_open_status std_logic std_logic_vector unsigned signed boolean_vector integer_vector std_ulogic std_ulogic_vector unresolved_unsigned u_unsigned unresolved_signed u_signedreal_vector time_vector",literal:"false true note warning error failure line text side width"},i:"{",c:[e.CBCM,e.C("--","$"),e.QSM,{cN:"number",b:o,r:0},{cN:"string",b:"'(U|X|0|1|Z|W|L|H|-)'",c:[e.BE]},{cN:"symbol",b:"'[A-Za-z](_?[A-Za-z0-9])*",c:[e.BE]}]}})),e.registerLanguage("vim",(function(e){return{l:/[!#@\w]+/,k:{keyword:"N|0 P|0 X|0 a|0 ab abc abo al am an|0 ar arga argd arge argdo argg argl argu as au aug aun b|0 bN ba bad bd be bel bf bl bm bn bo bp br brea breaka breakd breakl bro bufdo buffers bun bw c|0 cN cNf ca cabc caddb cad caddf cal cat cb cc ccl cd ce cex cf cfir cgetb cgete cg changes chd che checkt cl cla clo cm cmapc cme cn cnew cnf cno cnorea cnoreme co col colo com comc comp con conf cope cp cpf cq cr cs cst cu cuna cunme cw delm deb debugg delc delf dif diffg diffo diffp diffpu diffs diffthis dig di dl dell dj dli do doautoa dp dr ds dsp e|0 ea ec echoe echoh echom echon el elsei em en endfo endf endt endw ene ex exe exi exu f|0 files filet fin fina fini fir fix fo foldc foldd folddoc foldo for fu go gr grepa gu gv ha helpf helpg helpt hi hid his ia iabc if ij il im imapc ime ino inorea inoreme int is isp iu iuna iunme j|0 ju k|0 keepa kee keepj lN lNf l|0 lad laddb laddf la lan lat lb lc lch lcl lcs le lefta let lex lf lfir lgetb lgete lg lgr lgrepa lh ll lla lli lmak lm lmapc lne lnew lnf ln loadk lo loc lockv lol lope lp lpf lr ls lt lu lua luad luaf lv lvimgrepa lw m|0 ma mak map mapc marks mat me menut mes mk mks mksp mkv mkvie mod mz mzf nbc nb nbs new nm nmapc nme nn nnoreme noa no noh norea noreme norm nu nun nunme ol o|0 om omapc ome on ono onoreme opt ou ounme ow p|0 profd prof pro promptr pc ped pe perld po popu pp pre prev ps pt ptN ptf ptj ptl ptn ptp ptr pts pu pw py3 python3 py3d py3f py pyd pyf quita qa rec red redi redr redraws reg res ret retu rew ri rightb rub rubyd rubyf rund ru rv sN san sa sal sav sb sbN sba sbf sbl sbm sbn sbp sbr scrip scripte scs se setf setg setl sf sfir sh sim sig sil sl sla sm smap smapc sme sn sni sno snor snoreme sor so spelld spe spelli spellr spellu spellw sp spr sre st sta startg startr star stopi stj sts sun sunm sunme sus sv sw sy synti sync tN tabN tabc tabdo tabe tabf tabfir tabl tabm tabnew tabn tabo tabp tabr tabs tab ta tags tc tcld tclf te tf th tj tl tm tn to tp tr try ts tu u|0 undoj undol una unh unl unlo unm unme uns up ve verb vert vim vimgrepa vi viu vie vm vmapc vme vne vn vnoreme vs vu vunme windo w|0 wN wa wh wi winc winp wn wp wq wqa ws wu wv x|0 xa xmapc xm xme xn xnoreme xu xunme y|0 z|0 ~ Next Print append abbreviate abclear aboveleft all amenu anoremenu args argadd argdelete argedit argglobal arglocal argument ascii autocmd augroup aunmenu buffer bNext ball badd bdelete behave belowright bfirst blast bmodified bnext botright bprevious brewind break breakadd breakdel breaklist browse bunload bwipeout change cNext cNfile cabbrev cabclear caddbuffer caddexpr caddfile call catch cbuffer cclose center cexpr cfile cfirst cgetbuffer cgetexpr cgetfile chdir checkpath checktime clist clast close cmap cmapclear cmenu cnext cnewer cnfile cnoremap cnoreabbrev cnoremenu copy colder colorscheme command comclear compiler continue confirm copen cprevious cpfile cquit crewind cscope cstag cunmap cunabbrev cunmenu cwindow delete delmarks debug debuggreedy delcommand delfunction diffupdate diffget diffoff diffpatch diffput diffsplit digraphs display deletel djump dlist doautocmd doautoall deletep drop dsearch dsplit edit earlier echo echoerr echohl echomsg else elseif emenu endif endfor endfunction endtry endwhile enew execute exit exusage file filetype find finally finish first fixdel fold foldclose folddoopen folddoclosed foldopen function global goto grep grepadd gui gvim hardcopy help helpfind helpgrep helptags highlight hide history insert iabbrev iabclear ijump ilist imap imapclear imenu inoremap inoreabbrev inoremenu intro isearch isplit iunmap iunabbrev iunmenu join jumps keepalt keepmarks keepjumps lNext lNfile list laddexpr laddbuffer laddfile last language later lbuffer lcd lchdir lclose lcscope left leftabove lexpr lfile lfirst lgetbuffer lgetexpr lgetfile lgrep lgrepadd lhelpgrep llast llist lmake lmap lmapclear lnext lnewer lnfile lnoremap loadkeymap loadview lockmarks lockvar lolder lopen lprevious lpfile lrewind ltag lunmap luado luafile lvimgrep lvimgrepadd lwindow move mark make mapclear match menu menutranslate messages mkexrc mksession mkspell mkvimrc mkview mode mzscheme mzfile nbclose nbkey nbsart next nmap nmapclear nmenu nnoremap nnoremenu noautocmd noremap nohlsearch noreabbrev noremenu normal number nunmap nunmenu oldfiles open omap omapclear omenu only onoremap onoremenu options ounmap ounmenu ownsyntax print profdel profile promptfind promptrepl pclose pedit perl perldo pop popup ppop preserve previous psearch ptag ptNext ptfirst ptjump ptlast ptnext ptprevious ptrewind ptselect put pwd py3do py3file python pydo pyfile quit quitall qall read recover redo redir redraw redrawstatus registers resize retab return rewind right rightbelow ruby rubydo rubyfile rundo runtime rviminfo substitute sNext sandbox sargument sall saveas sbuffer sbNext sball sbfirst sblast sbmodified sbnext sbprevious sbrewind scriptnames scriptencoding scscope set setfiletype setglobal setlocal sfind sfirst shell simalt sign silent sleep slast smagic smapclear smenu snext sniff snomagic snoremap snoremenu sort source spelldump spellgood spellinfo spellrepall spellundo spellwrong split sprevious srewind stop stag startgreplace startreplace startinsert stopinsert stjump stselect sunhide sunmap sunmenu suspend sview swapname syntax syntime syncbind tNext tabNext tabclose tabedit tabfind tabfirst tablast tabmove tabnext tabonly tabprevious tabrewind tag tcl tcldo tclfile tearoff tfirst throw tjump tlast tmenu tnext topleft tprevious trewind tselect tunmenu undo undojoin undolist unabbreviate unhide unlet unlockvar unmap unmenu unsilent update vglobal version verbose vertical vimgrep vimgrepadd visual viusage view vmap vmapclear vmenu vnew vnoremap vnoremenu vsplit vunmap vunmenu write wNext wall while winsize wincmd winpos wnext wprevious wqall wsverb wundo wviminfo xit xall xmapclear xmap xmenu xnoremap xnoremenu xunmap xunmenu yank",built_in:"synIDtrans atan2 range matcharg did_filetype asin feedkeys xor argv complete_check add getwinposx getqflist getwinposy screencol clearmatches empty extend getcmdpos mzeval garbagecollect setreg ceil sqrt diff_hlID inputsecret get getfperm getpid filewritable shiftwidth max sinh isdirectory synID system inputrestore winline atan visualmode inputlist tabpagewinnr round getregtype mapcheck hasmapto histdel argidx findfile sha256 exists toupper getcmdline taglist string getmatches bufnr strftime winwidth bufexists strtrans tabpagebuflist setcmdpos remote_read printf setloclist getpos getline bufwinnr float2nr len getcmdtype diff_filler luaeval resolve libcallnr foldclosedend reverse filter has_key bufname str2float strlen setline getcharmod setbufvar index searchpos shellescape undofile foldclosed setqflist buflisted strchars str2nr virtcol floor remove undotree remote_expr winheight gettabwinvar reltime cursor tabpagenr finddir localtime acos getloclist search tanh matchend rename gettabvar strdisplaywidth type abs py3eval setwinvar tolower wildmenumode log10 spellsuggest bufloaded synconcealed nextnonblank server2client complete settabwinvar executable input wincol setmatches getftype hlID inputsave searchpair or screenrow line settabvar histadd deepcopy strpart remote_peek and eval getftime submatch screenchar winsaveview matchadd mkdir screenattr getfontname libcall reltimestr getfsize winnr invert pow getbufline byte2line soundfold repeat fnameescape tagfiles sin strwidth spellbadword trunc maparg log lispindent hostname setpos globpath remote_foreground getchar synIDattr fnamemodify cscope_connection stridx winbufnr indent min complete_add nr2char searchpairpos inputdialog values matchlist items hlexists strridx browsedir expand fmod pathshorten line2byte argc count getwinvar glob foldtextresult getreg foreground cosh matchdelete has char2nr simplify histget searchdecl iconv winrestcmd pumvisible writefile foldlevel haslocaldir keys cos matchstr foldtext histnr tan tempname getcwd byteidx getbufvar islocked escape eventhandler remote_send serverlist winrestview synstack pyeval prevnonblank readfile cindent filereadable changenr exp"},i:/;/,c:[e.NM,e.ASM,{cN:"string",b:/"(\\"|\n\\|[^"\n])*"/},e.C('"',"$"),{cN:"variable",b:/[bwtglsav]:[\w\d_]*/},{cN:"function",bK:"function function!",e:"$",r:0,c:[e.TM,{cN:"params",b:"\\(",e:"\\)"}]},{cN:"symbol",b:/<[\w-]+>/}]}})),e.registerLanguage("x86asm",(function(e){return{cI:!0,l:"[.%]?"+e.IR,k:{keyword:"lock rep repe repz repne repnz xaquire xrelease bnd nobnd aaa aad aam aas adc add and arpl bb0_reset bb1_reset bound bsf bsr bswap bt btc btr bts call cbw cdq cdqe clc cld cli clts cmc cmp cmpsb cmpsd cmpsq cmpsw cmpxchg cmpxchg486 cmpxchg8b cmpxchg16b cpuid cpu_read cpu_write cqo cwd cwde daa das dec div dmint emms enter equ f2xm1 fabs fadd faddp fbld fbstp fchs fclex fcmovb fcmovbe fcmove fcmovnb fcmovnbe fcmovne fcmovnu fcmovu fcom fcomi fcomip fcomp fcompp fcos fdecstp fdisi fdiv fdivp fdivr fdivrp femms feni ffree ffreep fiadd ficom ficomp fidiv fidivr fild fimul fincstp finit fist fistp fisttp fisub fisubr fld fld1 fldcw fldenv fldl2e fldl2t fldlg2 fldln2 fldpi fldz fmul fmulp fnclex fndisi fneni fninit fnop fnsave fnstcw fnstenv fnstsw fpatan fprem fprem1 fptan frndint frstor fsave fscale fsetpm fsin fsincos fsqrt fst fstcw fstenv fstp fstsw fsub fsubp fsubr fsubrp ftst fucom fucomi fucomip fucomp fucompp fxam fxch fxtract fyl2x fyl2xp1 hlt ibts icebp idiv imul in inc incbin insb insd insw int int01 int1 int03 int3 into invd invpcid invlpg invlpga iret iretd iretq iretw jcxz jecxz jrcxz jmp jmpe lahf lar lds lea leave les lfence lfs lgdt lgs lidt lldt lmsw loadall loadall286 lodsb lodsd lodsq lodsw loop loope loopne loopnz loopz lsl lss ltr mfence monitor mov movd movq movsb movsd movsq movsw movsx movsxd movzx mul mwait neg nop not or out outsb outsd outsw packssdw packsswb packuswb paddb paddd paddsb paddsiw paddsw paddusb paddusw paddw pand pandn pause paveb pavgusb pcmpeqb pcmpeqd pcmpeqw pcmpgtb pcmpgtd pcmpgtw pdistib pf2id pfacc pfadd pfcmpeq pfcmpge pfcmpgt pfmax pfmin pfmul pfrcp pfrcpit1 pfrcpit2 pfrsqit1 pfrsqrt pfsub pfsubr pi2fd pmachriw pmaddwd pmagw pmulhriw pmulhrwa pmulhrwc pmulhw pmullw pmvgezb pmvlzb pmvnzb pmvzb pop popa popad popaw popf popfd popfq popfw por prefetch prefetchw pslld psllq psllw psrad psraw psrld psrlq psrlw psubb psubd psubsb psubsiw psubsw psubusb psubusw psubw punpckhbw punpckhdq punpckhwd punpcklbw punpckldq punpcklwd push pusha pushad pushaw pushf pushfd pushfq pushfw pxor rcl rcr rdshr rdmsr rdpmc rdtsc rdtscp ret retf retn rol ror rdm rsdc rsldt rsm rsts sahf sal salc sar sbb scasb scasd scasq scasw sfence sgdt shl shld shr shrd sidt sldt skinit smi smint smintold smsw stc std sti stosb stosd stosq stosw str sub svdc svldt svts swapgs syscall sysenter sysexit sysret test ud0 ud1 ud2b ud2 ud2a umov verr verw fwait wbinvd wrshr wrmsr xadd xbts xchg xlatb xlat xor cmove cmovz cmovne cmovnz cmova cmovnbe cmovae cmovnb cmovb cmovnae cmovbe cmovna cmovg cmovnle cmovge cmovnl cmovl cmovnge cmovle cmovng cmovc cmovnc cmovo cmovno cmovs cmovns cmovp cmovpe cmovnp cmovpo je jz jne jnz ja jnbe jae jnb jb jnae jbe jna jg jnle jge jnl jl jnge jle jng jc jnc jo jno js jns jpo jnp jpe jp sete setz setne setnz seta setnbe setae setnb setnc setb setnae setcset setbe setna setg setnle setge setnl setl setnge setle setng sets setns seto setno setpe setp setpo setnp addps addss andnps andps cmpeqps cmpeqss cmpleps cmpless cmpltps cmpltss cmpneqps cmpneqss cmpnleps cmpnless cmpnltps cmpnltss cmpordps cmpordss cmpunordps cmpunordss cmpps cmpss comiss cvtpi2ps cvtps2pi cvtsi2ss cvtss2si cvttps2pi cvttss2si divps divss ldmxcsr maxps maxss minps minss movaps movhps movlhps movlps movhlps movmskps movntps movss movups mulps mulss orps rcpps rcpss rsqrtps rsqrtss shufps sqrtps sqrtss stmxcsr subps subss ucomiss unpckhps unpcklps xorps fxrstor fxrstor64 fxsave fxsave64 xgetbv xsetbv xsave xsave64 xsaveopt xsaveopt64 xrstor xrstor64 prefetchnta prefetcht0 prefetcht1 prefetcht2 maskmovq movntq pavgb pavgw pextrw pinsrw pmaxsw pmaxub pminsw pminub pmovmskb pmulhuw psadbw pshufw pf2iw pfnacc pfpnacc pi2fw pswapd maskmovdqu clflush movntdq movnti movntpd movdqa movdqu movdq2q movq2dq paddq pmuludq pshufd pshufhw pshuflw pslldq psrldq psubq punpckhqdq punpcklqdq addpd addsd andnpd andpd cmpeqpd cmpeqsd cmplepd cmplesd cmpltpd cmpltsd cmpneqpd cmpneqsd cmpnlepd cmpnlesd cmpnltpd cmpnltsd cmpordpd cmpordsd cmpunordpd cmpunordsd cmppd comisd cvtdq2pd cvtdq2ps cvtpd2dq cvtpd2pi cvtpd2ps cvtpi2pd cvtps2dq cvtps2pd cvtsd2si cvtsd2ss cvtsi2sd cvtss2sd cvttpd2pi cvttpd2dq cvttps2dq cvttsd2si divpd divsd maxpd maxsd minpd minsd movapd movhpd movlpd movmskpd movupd mulpd mulsd orpd shufpd sqrtpd sqrtsd subpd subsd ucomisd unpckhpd unpcklpd xorpd addsubpd addsubps haddpd haddps hsubpd hsubps lddqu movddup movshdup movsldup clgi stgi vmcall vmclear vmfunc vmlaunch vmload vmmcall vmptrld vmptrst vmread vmresume vmrun vmsave vmwrite vmxoff vmxon invept invvpid pabsb pabsw pabsd palignr phaddw phaddd phaddsw phsubw phsubd phsubsw pmaddubsw pmulhrsw pshufb psignb psignw psignd extrq insertq movntsd movntss lzcnt blendpd blendps blendvpd blendvps dppd dpps extractps insertps movntdqa mpsadbw packusdw pblendvb pblendw pcmpeqq pextrb pextrd pextrq phminposuw pinsrb pinsrd pinsrq pmaxsb pmaxsd pmaxud pmaxuw pminsb pminsd pminud pminuw pmovsxbw pmovsxbd pmovsxbq pmovsxwd pmovsxwq pmovsxdq pmovzxbw pmovzxbd pmovzxbq pmovzxwd pmovzxwq pmovzxdq pmuldq pmulld ptest roundpd roundps roundsd roundss crc32 pcmpestri pcmpestrm pcmpistri pcmpistrm pcmpgtq popcnt getsec pfrcpv pfrsqrtv movbe aesenc aesenclast aesdec aesdeclast aesimc aeskeygenassist vaesenc vaesenclast vaesdec vaesdeclast vaesimc vaeskeygenassist vaddpd vaddps vaddsd vaddss vaddsubpd vaddsubps vandpd vandps vandnpd vandnps vblendpd vblendps vblendvpd vblendvps vbroadcastss vbroadcastsd vbroadcastf128 vcmpeq_ospd vcmpeqpd vcmplt_ospd vcmpltpd vcmple_ospd vcmplepd vcmpunord_qpd vcmpunordpd vcmpneq_uqpd vcmpneqpd vcmpnlt_uspd vcmpnltpd vcmpnle_uspd vcmpnlepd vcmpord_qpd vcmpordpd vcmpeq_uqpd vcmpnge_uspd vcmpngepd vcmpngt_uspd vcmpngtpd vcmpfalse_oqpd vcmpfalsepd vcmpneq_oqpd vcmpge_ospd vcmpgepd vcmpgt_ospd vcmpgtpd vcmptrue_uqpd vcmptruepd vcmplt_oqpd vcmple_oqpd vcmpunord_spd vcmpneq_uspd vcmpnlt_uqpd vcmpnle_uqpd vcmpord_spd vcmpeq_uspd vcmpnge_uqpd vcmpngt_uqpd vcmpfalse_ospd vcmpneq_ospd vcmpge_oqpd vcmpgt_oqpd vcmptrue_uspd vcmppd vcmpeq_osps vcmpeqps vcmplt_osps vcmpltps vcmple_osps vcmpleps vcmpunord_qps vcmpunordps vcmpneq_uqps vcmpneqps vcmpnlt_usps vcmpnltps vcmpnle_usps vcmpnleps vcmpord_qps vcmpordps vcmpeq_uqps vcmpnge_usps vcmpngeps vcmpngt_usps vcmpngtps vcmpfalse_oqps vcmpfalseps vcmpneq_oqps vcmpge_osps vcmpgeps vcmpgt_osps vcmpgtps vcmptrue_uqps vcmptrueps vcmplt_oqps vcmple_oqps vcmpunord_sps vcmpneq_usps vcmpnlt_uqps vcmpnle_uqps vcmpord_sps vcmpeq_usps vcmpnge_uqps vcmpngt_uqps vcmpfalse_osps vcmpneq_osps vcmpge_oqps vcmpgt_oqps vcmptrue_usps vcmpps vcmpeq_ossd vcmpeqsd vcmplt_ossd vcmpltsd vcmple_ossd vcmplesd vcmpunord_qsd vcmpunordsd vcmpneq_uqsd vcmpneqsd vcmpnlt_ussd vcmpnltsd vcmpnle_ussd vcmpnlesd vcmpord_qsd vcmpordsd vcmpeq_uqsd vcmpnge_ussd vcmpngesd vcmpngt_ussd vcmpngtsd vcmpfalse_oqsd vcmpfalsesd vcmpneq_oqsd vcmpge_ossd vcmpgesd vcmpgt_ossd vcmpgtsd vcmptrue_uqsd vcmptruesd vcmplt_oqsd vcmple_oqsd vcmpunord_ssd vcmpneq_ussd vcmpnlt_uqsd vcmpnle_uqsd vcmpord_ssd vcmpeq_ussd vcmpnge_uqsd vcmpngt_uqsd vcmpfalse_ossd vcmpneq_ossd vcmpge_oqsd vcmpgt_oqsd vcmptrue_ussd vcmpsd vcmpeq_osss vcmpeqss vcmplt_osss vcmpltss vcmple_osss vcmpless vcmpunord_qss vcmpunordss vcmpneq_uqss vcmpneqss vcmpnlt_usss vcmpnltss vcmpnle_usss vcmpnless vcmpord_qss vcmpordss vcmpeq_uqss vcmpnge_usss vcmpngess vcmpngt_usss vcmpngtss vcmpfalse_oqss vcmpfalsess vcmpneq_oqss vcmpge_osss vcmpgess vcmpgt_osss vcmpgtss vcmptrue_uqss vcmptruess vcmplt_oqss vcmple_oqss vcmpunord_sss vcmpneq_usss vcmpnlt_uqss vcmpnle_uqss vcmpord_sss vcmpeq_usss vcmpnge_uqss vcmpngt_uqss vcmpfalse_osss vcmpneq_osss vcmpge_oqss vcmpgt_oqss vcmptrue_usss vcmpss vcomisd vcomiss vcvtdq2pd vcvtdq2ps vcvtpd2dq vcvtpd2ps vcvtps2dq vcvtps2pd vcvtsd2si vcvtsd2ss vcvtsi2sd vcvtsi2ss vcvtss2sd vcvtss2si vcvttpd2dq vcvttps2dq vcvttsd2si vcvttss2si vdivpd vdivps vdivsd vdivss vdppd vdpps vextractf128 vextractps vhaddpd vhaddps vhsubpd vhsubps vinsertf128 vinsertps vlddqu vldqqu vldmxcsr vmaskmovdqu vmaskmovps vmaskmovpd vmaxpd vmaxps vmaxsd vmaxss vminpd vminps vminsd vminss vmovapd vmovaps vmovd vmovq vmovddup vmovdqa vmovqqa vmovdqu vmovqqu vmovhlps vmovhpd vmovhps vmovlhps vmovlpd vmovlps vmovmskpd vmovmskps vmovntdq vmovntqq vmovntdqa vmovntpd vmovntps vmovsd vmovshdup vmovsldup vmovss vmovupd vmovups vmpsadbw vmulpd vmulps vmulsd vmulss vorpd vorps vpabsb vpabsw vpabsd vpacksswb vpackssdw vpackuswb vpackusdw vpaddb vpaddw vpaddd vpaddq vpaddsb vpaddsw vpaddusb vpaddusw vpalignr vpand vpandn vpavgb vpavgw vpblendvb vpblendw vpcmpestri vpcmpestrm vpcmpistri vpcmpistrm vpcmpeqb vpcmpeqw vpcmpeqd vpcmpeqq vpcmpgtb vpcmpgtw vpcmpgtd vpcmpgtq vpermilpd vpermilps vperm2f128 vpextrb vpextrw vpextrd vpextrq vphaddw vphaddd vphaddsw vphminposuw vphsubw vphsubd vphsubsw vpinsrb vpinsrw vpinsrd vpinsrq vpmaddwd vpmaddubsw vpmaxsb vpmaxsw vpmaxsd vpmaxub vpmaxuw vpmaxud vpminsb vpminsw vpminsd vpminub vpminuw vpminud vpmovmskb vpmovsxbw vpmovsxbd vpmovsxbq vpmovsxwd vpmovsxwq vpmovsxdq vpmovzxbw vpmovzxbd vpmovzxbq vpmovzxwd vpmovzxwq vpmovzxdq vpmulhuw vpmulhrsw vpmulhw vpmullw vpmulld vpmuludq vpmuldq vpor vpsadbw vpshufb vpshufd vpshufhw vpshuflw vpsignb vpsignw vpsignd vpslldq vpsrldq vpsllw vpslld vpsllq vpsraw vpsrad vpsrlw vpsrld vpsrlq vptest vpsubb vpsubw vpsubd vpsubq vpsubsb vpsubsw vpsubusb vpsubusw vpunpckhbw vpunpckhwd vpunpckhdq vpunpckhqdq vpunpcklbw vpunpcklwd vpunpckldq vpunpcklqdq vpxor vrcpps vrcpss vrsqrtps vrsqrtss vroundpd vroundps vroundsd vroundss vshufpd vshufps vsqrtpd vsqrtps vsqrtsd vsqrtss vstmxcsr vsubpd vsubps vsubsd vsubss vtestps vtestpd vucomisd vucomiss vunpckhpd vunpckhps vunpcklpd vunpcklps vxorpd vxorps vzeroall vzeroupper pclmullqlqdq pclmulhqlqdq pclmullqhqdq pclmulhqhqdq pclmulqdq vpclmullqlqdq vpclmulhqlqdq vpclmullqhqdq vpclmulhqhqdq vpclmulqdq vfmadd132ps vfmadd132pd vfmadd312ps vfmadd312pd vfmadd213ps vfmadd213pd vfmadd123ps vfmadd123pd vfmadd231ps vfmadd231pd vfmadd321ps vfmadd321pd vfmaddsub132ps vfmaddsub132pd vfmaddsub312ps vfmaddsub312pd vfmaddsub213ps vfmaddsub213pd vfmaddsub123ps vfmaddsub123pd vfmaddsub231ps vfmaddsub231pd vfmaddsub321ps vfmaddsub321pd vfmsub132ps vfmsub132pd vfmsub312ps vfmsub312pd vfmsub213ps vfmsub213pd vfmsub123ps vfmsub123pd vfmsub231ps vfmsub231pd vfmsub321ps vfmsub321pd vfmsubadd132ps vfmsubadd132pd vfmsubadd312ps vfmsubadd312pd vfmsubadd213ps vfmsubadd213pd vfmsubadd123ps vfmsubadd123pd vfmsubadd231ps vfmsubadd231pd vfmsubadd321ps vfmsubadd321pd vfnmadd132ps vfnmadd132pd vfnmadd312ps vfnmadd312pd vfnmadd213ps vfnmadd213pd vfnmadd123ps vfnmadd123pd vfnmadd231ps vfnmadd231pd vfnmadd321ps vfnmadd321pd vfnmsub132ps vfnmsub132pd vfnmsub312ps vfnmsub312pd vfnmsub213ps vfnmsub213pd vfnmsub123ps vfnmsub123pd vfnmsub231ps vfnmsub231pd vfnmsub321ps vfnmsub321pd vfmadd132ss vfmadd132sd vfmadd312ss vfmadd312sd vfmadd213ss vfmadd213sd vfmadd123ss vfmadd123sd vfmadd231ss vfmadd231sd vfmadd321ss vfmadd321sd vfmsub132ss vfmsub132sd vfmsub312ss vfmsub312sd vfmsub213ss vfmsub213sd vfmsub123ss vfmsub123sd vfmsub231ss vfmsub231sd vfmsub321ss vfmsub321sd vfnmadd132ss vfnmadd132sd vfnmadd312ss vfnmadd312sd vfnmadd213ss vfnmadd213sd vfnmadd123ss vfnmadd123sd vfnmadd231ss vfnmadd231sd vfnmadd321ss vfnmadd321sd vfnmsub132ss vfnmsub132sd vfnmsub312ss vfnmsub312sd vfnmsub213ss vfnmsub213sd vfnmsub123ss vfnmsub123sd vfnmsub231ss vfnmsub231sd vfnmsub321ss vfnmsub321sd rdfsbase rdgsbase rdrand wrfsbase wrgsbase vcvtph2ps vcvtps2ph adcx adox rdseed clac stac xstore xcryptecb xcryptcbc xcryptctr xcryptcfb xcryptofb montmul xsha1 xsha256 llwpcb slwpcb lwpval lwpins vfmaddpd vfmaddps vfmaddsd vfmaddss vfmaddsubpd vfmaddsubps vfmsubaddpd vfmsubaddps vfmsubpd vfmsubps vfmsubsd vfmsubss vfnmaddpd vfnmaddps vfnmaddsd vfnmaddss vfnmsubpd vfnmsubps vfnmsubsd vfnmsubss vfrczpd vfrczps vfrczsd vfrczss vpcmov vpcomb vpcomd vpcomq vpcomub vpcomud vpcomuq vpcomuw vpcomw vphaddbd vphaddbq vphaddbw vphadddq vphaddubd vphaddubq vphaddubw vphaddudq vphadduwd vphadduwq vphaddwd vphaddwq vphsubbw vphsubdq vphsubwd vpmacsdd vpmacsdqh vpmacsdql vpmacssdd vpmacssdqh vpmacssdql vpmacsswd vpmacssww vpmacswd vpmacsww vpmadcsswd vpmadcswd vpperm vprotb vprotd vprotq vprotw vpshab vpshad vpshaq vpshaw vpshlb vpshld vpshlq vpshlw vbroadcasti128 vpblendd vpbroadcastb vpbroadcastw vpbroadcastd vpbroadcastq vpermd vpermpd vpermps vpermq vperm2i128 vextracti128 vinserti128 vpmaskmovd vpmaskmovq vpsllvd vpsllvq vpsravd vpsrlvd vpsrlvq vgatherdpd vgatherqpd vgatherdps vgatherqps vpgatherdd vpgatherqd vpgatherdq vpgatherqq xabort xbegin xend xtest andn bextr blci blcic blsi blsic blcfill blsfill blcmsk blsmsk blsr blcs bzhi mulx pdep pext rorx sarx shlx shrx tzcnt tzmsk t1mskc valignd valignq vblendmpd vblendmps vbroadcastf32x4 vbroadcastf64x4 vbroadcasti32x4 vbroadcasti64x4 vcompresspd vcompressps vcvtpd2udq vcvtps2udq vcvtsd2usi vcvtss2usi vcvttpd2udq vcvttps2udq vcvttsd2usi vcvttss2usi vcvtudq2pd vcvtudq2ps vcvtusi2sd vcvtusi2ss vexpandpd vexpandps vextractf32x4 vextractf64x4 vextracti32x4 vextracti64x4 vfixupimmpd vfixupimmps vfixupimmsd vfixupimmss vgetexppd vgetexpps vgetexpsd vgetexpss vgetmantpd vgetmantps vgetmantsd vgetmantss vinsertf32x4 vinsertf64x4 vinserti32x4 vinserti64x4 vmovdqa32 vmovdqa64 vmovdqu32 vmovdqu64 vpabsq vpandd vpandnd vpandnq vpandq vpblendmd vpblendmq vpcmpltd vpcmpled vpcmpneqd vpcmpnltd vpcmpnled vpcmpd vpcmpltq vpcmpleq vpcmpneqq vpcmpnltq vpcmpnleq vpcmpq vpcmpequd vpcmpltud vpcmpleud vpcmpnequd vpcmpnltud vpcmpnleud vpcmpud vpcmpequq vpcmpltuq vpcmpleuq vpcmpnequq vpcmpnltuq vpcmpnleuq vpcmpuq vpcompressd vpcompressq vpermi2d vpermi2pd vpermi2ps vpermi2q vpermt2d vpermt2pd vpermt2ps vpermt2q vpexpandd vpexpandq vpmaxsq vpmaxuq vpminsq vpminuq vpmovdb vpmovdw vpmovqb vpmovqd vpmovqw vpmovsdb vpmovsdw vpmovsqb vpmovsqd vpmovsqw vpmovusdb vpmovusdw vpmovusqb vpmovusqd vpmovusqw vpord vporq vprold vprolq vprolvd vprolvq vprord vprorq vprorvd vprorvq vpscatterdd vpscatterdq vpscatterqd vpscatterqq vpsraq vpsravq vpternlogd vpternlogq vptestmd vptestmq vptestnmd vptestnmq vpxord vpxorq vrcp14pd vrcp14ps vrcp14sd vrcp14ss vrndscalepd vrndscaleps vrndscalesd vrndscaless vrsqrt14pd vrsqrt14ps vrsqrt14sd vrsqrt14ss vscalefpd vscalefps vscalefsd vscalefss vscatterdpd vscatterdps vscatterqpd vscatterqps vshuff32x4 vshuff64x2 vshufi32x4 vshufi64x2 kandnw kandw kmovw knotw kortestw korw kshiftlw kshiftrw kunpckbw kxnorw kxorw vpbroadcastmb2q vpbroadcastmw2d vpconflictd vpconflictq vplzcntd vplzcntq vexp2pd vexp2ps vrcp28pd vrcp28ps vrcp28sd vrcp28ss vrsqrt28pd vrsqrt28ps vrsqrt28sd vrsqrt28ss vgatherpf0dpd vgatherpf0dps vgatherpf0qpd vgatherpf0qps vgatherpf1dpd vgatherpf1dps vgatherpf1qpd vgatherpf1qps vscatterpf0dpd vscatterpf0dps vscatterpf0qpd vscatterpf0qps vscatterpf1dpd vscatterpf1dps vscatterpf1qpd vscatterpf1qps prefetchwt1 bndmk bndcl bndcu bndcn bndmov bndldx bndstx sha1rnds4 sha1nexte sha1msg1 sha1msg2 sha256rnds2 sha256msg1 sha256msg2 hint_nop0 hint_nop1 hint_nop2 hint_nop3 hint_nop4 hint_nop5 hint_nop6 hint_nop7 hint_nop8 hint_nop9 hint_nop10 hint_nop11 hint_nop12 hint_nop13 hint_nop14 hint_nop15 hint_nop16 hint_nop17 hint_nop18 hint_nop19 hint_nop20 hint_nop21 hint_nop22 hint_nop23 hint_nop24 hint_nop25 hint_nop26 hint_nop27 hint_nop28 hint_nop29 hint_nop30 hint_nop31 hint_nop32 hint_nop33 hint_nop34 hint_nop35 hint_nop36 hint_nop37 hint_nop38 hint_nop39 hint_nop40 hint_nop41 hint_nop42 hint_nop43 hint_nop44 hint_nop45 hint_nop46 hint_nop47 hint_nop48 hint_nop49 hint_nop50 hint_nop51 hint_nop52 hint_nop53 hint_nop54 hint_nop55 hint_nop56 hint_nop57 hint_nop58 hint_nop59 hint_nop60 hint_nop61 hint_nop62 hint_nop63",built_in:"ip eip rip al ah bl bh cl ch dl dh sil dil bpl spl r8b r9b r10b r11b r12b r13b r14b r15b ax bx cx dx si di bp sp r8w r9w r10w r11w r12w r13w r14w r15w eax ebx ecx edx esi edi ebp esp eip r8d r9d r10d r11d r12d r13d r14d r15d rax rbx rcx rdx rsi rdi rbp rsp r8 r9 r10 r11 r12 r13 r14 r15 cs ds es fs gs ss st st0 st1 st2 st3 st4 st5 st6 st7 mm0 mm1 mm2 mm3 mm4 mm5 mm6 mm7 xmm0 xmm1 xmm2 xmm3 xmm4 xmm5 xmm6 xmm7 xmm8 xmm9 xmm10 xmm11 xmm12 xmm13 xmm14 xmm15 xmm16 xmm17 xmm18 xmm19 xmm20 xmm21 xmm22 xmm23 xmm24 xmm25 xmm26 xmm27 xmm28 xmm29 xmm30 xmm31 ymm0 ymm1 ymm2 ymm3 ymm4 ymm5 ymm6 ymm7 ymm8 ymm9 ymm10 ymm11 ymm12 ymm13 ymm14 ymm15 ymm16 ymm17 ymm18 ymm19 ymm20 ymm21 ymm22 ymm23 ymm24 ymm25 ymm26 ymm27 ymm28 ymm29 ymm30 ymm31 zmm0 zmm1 zmm2 zmm3 zmm4 zmm5 zmm6 zmm7 zmm8 zmm9 zmm10 zmm11 zmm12 zmm13 zmm14 zmm15 zmm16 zmm17 zmm18 zmm19 zmm20 zmm21 zmm22 zmm23 zmm24 zmm25 zmm26 zmm27 zmm28 zmm29 zmm30 zmm31 k0 k1 k2 k3 k4 k5 k6 k7 bnd0 bnd1 bnd2 bnd3 cr0 cr1 cr2 cr3 cr4 cr8 dr0 dr1 dr2 dr3 dr8 tr3 tr4 tr5 tr6 tr7 r0 r1 r2 r3 r4 r5 r6 r7 r0b r1b r2b r3b r4b r5b r6b r7b r0w r1w r2w r3w r4w r5w r6w r7w r0d r1d r2d r3d r4d r5d r6d r7d r0h r1h r2h r3h r0l r1l r2l r3l r4l r5l r6l r7l r8l r9l r10l r11l r12l r13l r14l r15l db dw dd dq dt ddq do dy dz resb resw resd resq rest resdq reso resy resz incbin equ times byte word dword qword nosplit rel abs seg wrt strict near far a32 ptr",meta:"%define %xdefine %+ %undef %defstr %deftok %assign %strcat %strlen %substr %rotate %elif %else %endif %if %ifmacro %ifctx %ifidn %ifidni %ifid %ifnum %ifstr %iftoken %ifempty %ifenv %error %warning %fatal %rep %endrep %include %push %pop %repl %pathsearch %depend %use %arg %stacksize %local %line %comment %endcomment .nolist __FILE__ __LINE__ __SECT__ __BITS__ __OUTPUT_FORMAT__ __DATE__ __TIME__ __DATE_NUM__ __TIME_NUM__ __UTC_DATE__ __UTC_TIME__ __UTC_DATE_NUM__ __UTC_TIME_NUM__ __PASS__ struc endstruc istruc at iend align alignb sectalign daz nodaz up down zero default option assume public bits use16 use32 use64 default section segment absolute extern global common cpu float __utf16__ __utf16le__ __utf16be__ __utf32__ __utf32le__ __utf32be__ __float8__ __float16__ __float32__ __float64__ __float80m__ __float80e__ __float128l__ __float128h__ __Infinity__ __QNaN__ __SNaN__ Inf NaN QNaN SNaN float8 float16 float32 float64 float80m float80e float128l float128h __FLOAT_DAZ__ __FLOAT_ROUND__ __FLOAT__"},c:[e.C(";","$",{r:0}),{cN:"number",v:[{b:"\\b(?:([0-9][0-9_]*)?\\.[0-9_]*(?:[eE][+-]?[0-9_]+)?|(0[Xx])?[0-9][0-9_]*\\.?[0-9_]*(?:[pP](?:[+-]?[0-9_]+)?)?)\\b",r:0},{b:"\\$[0-9][0-9A-Fa-f]*",r:0},{b:"\\b(?:[0-9A-Fa-f][0-9A-Fa-f_]*[Hh]|[0-9][0-9_]*[DdTt]?|[0-7][0-7_]*[QqOo]|[0-1][0-1_]*[BbYy])\\b"},{b:"\\b(?:0[Xx][0-9A-Fa-f_]+|0[DdTt][0-9_]+|0[QqOo][0-7_]+|0[BbYy][0-1_]+)\\b"}]},e.QSM,{cN:"string",v:[{b:"'",e:"[^\\\\]'"},{b:"`",e:"[^\\\\]`"}],r:0},{cN:"symbol",v:[{b:"^\\s*[A-Za-z._?][A-Za-z0-9_$#@~.?]*(:|\\s+label)"},{b:"^\\s*%%[A-Za-z0-9_$#@~.?]*:"}],r:0},{cN:"subst",b:"%[0-9]+",r:0},{cN:"subst",b:"%!S+",r:0},{cN:"meta",b:/^\s*\.[\w_-]+/}]}})),e.registerLanguage("xl",(function(e){var t="ObjectLoader Animate MovieCredits Slides Filters Shading Materials LensFlare Mapping VLCAudioVideo StereoDecoder PointCloud NetworkAccess RemoteControl RegExp ChromaKey Snowfall NodeJS Speech Charts",r={keyword:"if then else do while until for loop import with is as where when by data constant integer real text name boolean symbol infix prefix postfix block tree",literal:"true false nil",built_in:"in mod rem and or xor not abs sign floor ceil sqrt sin cos tan asin acos atan exp expm1 log log2 log10 log1p pi at text_length text_range text_find text_replace contains page slide basic_slide title_slide title subtitle fade_in fade_out fade_at clear_color color line_color line_width texture_wrap texture_transform texture scale_?x scale_?y scale_?z? translate_?x translate_?y translate_?z? rotate_?x rotate_?y rotate_?z? rectangle circle ellipse sphere path line_to move_to quad_to curve_to theme background contents locally time mouse_?x mouse_?y mouse_buttons "+t},a={cN:"string",b:'"',e:'"',i:"\\n"},i={cN:"string",b:"'",e:"'",i:"\\n"},n={cN:"string",b:"<<",e:">>"},o={cN:"number",b:"[0-9]+#[0-9A-Z_]+(\\.[0-9-A-Z_]+)?#?([Ee][+-]?[0-9]+)?"},s={bK:"import",e:"$",k:r,c:[a]},l={cN:"function",b:/[a-z][^\n]*->/,rB:!0,e:/->/,c:[e.inherit(e.TM,{starts:{eW:!0,k:r}})]};return{aliases:["tao"],l:/[a-zA-Z][a-zA-Z0-9_?]*/,k:r,c:[e.CLCM,e.CBCM,a,i,n,l,s,o,e.NM]}})),e.registerLanguage("xquery",(function(e){var t="for let if while then else return where group by xquery encoding versionmodule namespace boundary-space preserve strip default collation base-uri orderingcopy-namespaces order declare import schema namespace function option in allowing emptyat tumbling window sliding window start when only end when previous next stable ascendingdescending empty greatest least some every satisfies switch case typeswitch try catch andor to union intersect instance of treat as castable cast map array delete insert intoreplace value rename copy modify update",r="false true xs:string xs:integer element item xs:date xs:datetime xs:float xs:double xs:decimal QName xs:anyURI xs:long xs:int xs:short xs:byte attribute",a={b:/\$[a-zA-Z0-9\-]+/},i={cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},n={cN:"string",v:[{b:/"/,e:/"/,c:[{b:/""/,r:0}]},{b:/'/,e:/'/,c:[{b:/''/,r:0}]}]},o={cN:"meta",b:"%\\w+"},s={cN:"comment",b:"\\(:",e:":\\)",r:10,c:[{cN:"doctag",b:"@\\w+"}]},l={b:"{",e:"}"},c=[a,n,i,s,o,l];return l.c=c,{aliases:["xpath","xq"],cI:!1,l:/[a-zA-Z\$][a-zA-Z0-9_:\-]*/,i:/(proc)|(abstract)|(extends)|(until)|(#)/,k:{keyword:t,literal:r},c:c}})),e.registerLanguage("zephir",(function(e){var t={cN:"string",c:[e.BE],v:[{b:'b"',e:'"'},{b:"b'",e:"'"},e.inherit(e.ASM,{i:null}),e.inherit(e.QSM,{i:null})]},r={v:[e.BNM,e.CNM]};return{aliases:["zep"],cI:!0,k:"and include_once list abstract global private echo interface as static endswitch array null if endwhile or const for endforeach self var let while isset public protected exit foreach throw elseif include __FILE__ empty require_once do xor return parent clone use __CLASS__ __LINE__ else break print eval new catch __METHOD__ case exception default die require __FUNCTION__ enddeclare final try switch continue endfor endif declare unset true false trait goto instanceof insteadof __DIR__ __NAMESPACE__ yield finally int uint long ulong char uchar double float bool boolean stringlikely unlikely",c:[e.CLCM,e.HCM,e.C("/\\*","\\*/",{c:[{cN:"doctag",b:"@[A-Za-z]+"}]}),e.C("__halt_compiler.+?;",!1,{eW:!0,k:"__halt_compiler",l:e.UIR}),{cN:"string",b:"<<<['\"]?\\w+['\"]?$",e:"^\\w+;",c:[e.BE]},{b:/(::|->)+[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/},{cN:"function",bK:"function",e:/[;{]/,eE:!0,i:"\\$|\\[|%",c:[e.UTM,{cN:"params",b:"\\(",e:"\\)",c:["self",e.CBCM,t,r]}]},{cN:"class",bK:"class interface",e:"{",eE:!0,i:/[:\(\$"]/,c:[{bK:"extends implements"},e.UTM]},{bK:"namespace",e:";",i:/[\.']/,c:[e.UTM]},{bK:"use",e:";",c:[e.UTM]},{b:"=>"},t,r]}})),e}));!function(){function t(t,r){var a,o={};if("tld?"!==t){if(r=r||window.location.toString(),!t)return r;if(t=t.toString(),a=r.match(/^mailto:([^\/].+)/))o.protocol="mailto",o.email=a[1];else{if((a=r.match(/(.*?)\/#\!(.*)/))&&(r=a[1]+a[2]),(a=r.match(/(.*?)#(.*)/))&&(o.hash=a[2],r=a[1]),o.hash&&t.match(/^#/))return h(t,o.hash);if((a=r.match(/(.*?)\?(.*)/))&&(o.query=a[2],r=a[1]),o.query&&t.match(/^\?/))return h(t,o.query);if((a=r.match(/(.*?)\:?\/\/(.*)/))&&(o.protocol=a[1].toLowerCase(),r=a[2]),(a=r.match(/(.*?)(\/.*)/))&&(o.path=a[2],r=a[1]),o.path=(o.path||"").replace(/^([^\/])/,"/$1"),t.match(/^[\-0-9]+$/)&&(t=t.replace(/^([^\/])/,"/$1")),t.match(/^\//))return e(t,o.path.substring(1));if((a=(a=e("/-1",o.path.substring(1)))&&a.match(/(.*?)\.([^.]+)$/))&&(o.file=a[0],o.filename=a[1],o.fileext=a[2]),(a=r.match(/(.*)\:([0-9]+)$/))&&(o.port=a[2],r=a[1]),(a=r.match(/(.*?)@(.*)/))&&(o.auth=a[1],r=a[2]),o.auth&&(a=o.auth.match(/(.*)\:(.*)/),o.user=a?a[1]:o.auth,o.pass=a?a[2]:void 0),o.hostname=r.toLowerCase(),"."===t.charAt(0))return e(t,o.hostname);o.port=o.port||("https"===o.protocol?"443":"80"),o.protocol=o.protocol||("443"===o.port?"https":"http")}return t in o?o[t]:"{}"===t?o:void 0}}function e(t,r){var a=t.charAt(0),o=r.split(a);return a===t?o:o[(t=parseInt(t.substring(1),10))<0?o.length+t:t-1]}function h(t,r){for(var a,o=t.charAt(0),e=r.split("&"),h=[],n={},c=[],i=t.substring(1),p=0,u=e.length;pthis.options.totalPages)throw new Error("Start page option is incorrect");if(this.options.totalPages=parseInt(this.options.totalPages),isNaN(this.options.totalPages))throw new Error("Total pages option is not correct!");if(this.options.visiblePages=parseInt(this.options.visiblePages),isNaN(this.options.visiblePages))throw new Error("Visible pages option is not correct!");if(this.options.totalPages"),this.$listContainer.addClass(this.options.paginationClass),"UL"!==g&&this.$element.append(this.$listContainer),this.render(this.getPages(this.options.startPage)),this.setupEvents(),this.options.initiateStartPageClick&&this.$element.trigger("page",this.options.startPage),this};f.prototype={constructor:f,destroy:function(){return this.$element.empty(),this.$element.removeData("twbs-pagination"),this.$element.off("page"),this},show:function(a){if(1>a||a>this.options.totalPages)throw new Error("Page is incorrect.");return this.render(this.getPages(a)),this.setupEvents(),this.$element.trigger("page",a),this},buildListItems:function(a){var b=[];if(this.options.first&&b.push(this.buildItem("first",1)),this.options.prev){var c=a.currentPage>1?a.currentPage-1:this.options.loop?this.options.totalPages:1;b.push(this.buildItem("prev",c))}for(var d=0;d"),e=a(""),f=null;switch(b){case"page":f=c,d.addClass(this.options.pageClass);break;case"first":f=this.options.first,d.addClass(this.options.firstClass);break;case"prev":f=this.options.prev,d.addClass(this.options.prevClass);break;case"next":f=this.options.next,d.addClass(this.options.nextClass);break;case"last":f=this.options.last,d.addClass(this.options.lastClass)}return d.data("page",c),d.data("page-type",b),d.append(e.attr("href",this.makeHref(c)).html(f)),d},getPages:function(a){var b=[],c=Math.floor(this.options.visiblePages/2),d=a-c+1-this.options.visiblePages%2,e=a+c;0>=d&&(d=1,e=this.options.visiblePages),e>this.options.totalPages&&(d=this.options.totalPages-this.options.visiblePages+1,e=this.options.totalPages);for(var f=d;e>=f;)b.push(f),f++;return{currentPage:a,numeric:b}},render:function(b){var c=this;this.$listContainer.children().remove(),this.$listContainer.append(this.buildListItems(b)),this.$listContainer.children().each((function(){var d=a(this),e=d.data("page-type");switch(e){case"page":d.data("page")===b.currentPage&&d.addClass(c.options.activeClass);break;case"first":d.toggleClass(c.options.disabledClass,1===b.currentPage);break;case"last":d.toggleClass(c.options.disabledClass,b.currentPage===c.options.totalPages);break;case"prev":d.toggleClass(c.options.disabledClass,!c.options.loop&&1===b.currentPage);break;case"next":d.toggleClass(c.options.disabledClass,!c.options.loop&&b.currentPage===c.options.totalPages)}}))},setupEvents:function(){var b=this;this.$listContainer.find("li").each((function(){var c=a(this);return c.off(),c.hasClass(b.options.disabledClass)||c.hasClass(b.options.activeClass)?void c.on("click",!1):void c.click((function(a){!b.options.href&&a.preventDefault(),b.show(parseInt(c.data("page")))}))}))},makeHref:function(a){return this.options.href?this.options.href.replace(this.options.hrefVariable,a):"#"}},a.fn.twbsPagination=function(b){var c,e=Array.prototype.slice.call(arguments,1),g=a(this),h=g.data("twbs-pagination"),i="object"==typeof b&&b;return h||g.data("twbs-pagination",h=new f(this,i)),"string"==typeof b&&(c=h[b].apply(h,e)),c===d?g:c},a.fn.twbsPagination.defaults={totalPages:0,startPage:1,visiblePages:5,initiateStartPageClick:!0,href:!1,hrefVariable:"{{number}}",first:"First",prev:"Previous",next:"Next",last:"Last",loop:!1,onPageClick:null,paginationClass:"pagination",nextClass:"next",prevClass:"prev",lastClass:"last",firstClass:"first",pageClass:"page",activeClass:"active",disabledClass:"disabled"},a.fn.twbsPagination.Constructor=f,a.fn.twbsPagination.noConflict=function(){return a.fn.twbsPagination=e,this}}(window.jQuery,window,document);!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t(require("jquery")):"function"==typeof define&&define.amd?define(["jquery"],t):e.Mark=t(e.jQuery)}(this,(function(e){"use strict";e=e&&e.hasOwnProperty("default")?e.default:e;var t="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},n=function(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")},r=function(){function e(e,t){for(var n=0;n1&&void 0!==arguments[1])||arguments[1],i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:[],o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:5e3;n(this,e),this.ctx=t,this.iframes=r,this.exclude=i,this.iframesTimeout=o}return r(e,[{key:"getContexts",value:function(){var e=[];return(void 0!==this.ctx&&this.ctx?NodeList.prototype.isPrototypeOf(this.ctx)?Array.prototype.slice.call(this.ctx):Array.isArray(this.ctx)?this.ctx:"string"==typeof this.ctx?Array.prototype.slice.call(document.querySelectorAll(this.ctx)):[this.ctx]:[]).forEach((function(t){var n=e.filter((function(e){return e.contains(t)})).length>0;-1!==e.indexOf(t)||n||e.push(t)})),e}},{key:"getIframeContents",value:function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:function(){},r=void 0;try{var i=e.contentWindow;if(r=i.document,!i||!r)throw new Error("iframe inaccessible")}catch(e){n()}r&&t(r)}},{key:"isIframeBlank",value:function(e){var t=e.getAttribute("src").trim();return"about:blank"===e.contentWindow.location.href&&"about:blank"!==t&&t}},{key:"observeIframeLoad",value:function(e,t,n){var r=this,i=!1,o=null,a=function a(){if(!i){i=!0,clearTimeout(o);try{r.isIframeBlank(e)||(e.removeEventListener("load",a),r.getIframeContents(e,t,n))}catch(e){n()}}};e.addEventListener("load",a),o=setTimeout(a,this.iframesTimeout)}},{key:"onIframeReady",value:function(e,t,n){try{"complete"===e.contentWindow.document.readyState?this.isIframeBlank(e)?this.observeIframeLoad(e,t,n):this.getIframeContents(e,t,n):this.observeIframeLoad(e,t,n)}catch(e){n()}}},{key:"waitForIframes",value:function(e,t){var n=this,r=0;this.forEachIframe(e,(function(){return!0}),(function(e){r++,n.waitForIframes(e.querySelector("html"),(function(){--r||t()}))}),(function(e){e||t()}))}},{key:"forEachIframe",value:function(t,n,r){var i=this,o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:function(){},a=t.querySelectorAll("iframe"),s=a.length,c=0;a=Array.prototype.slice.call(a);var u=function(){--s<=0&&o(c)};s||u(),a.forEach((function(t){e.matches(t,i.exclude)?u():i.onIframeReady(t,(function(e){n(t)&&(c++,r(e)),u()}),u)}))}},{key:"createIterator",value:function(e,t,n){return document.createNodeIterator(e,t,n,!1)}},{key:"createInstanceOnIframe",value:function(t){return new e(t.querySelector("html"),this.iframes)}},{key:"compareNodeIframe",value:function(e,t,n){if(e.compareDocumentPosition(n)&Node.DOCUMENT_POSITION_PRECEDING){if(null===t)return!0;if(t.compareDocumentPosition(n)&Node.DOCUMENT_POSITION_FOLLOWING)return!0}return!1}},{key:"getIteratorNode",value:function(e){var t=e.previousNode();return{prevNode:t,node:null===t?e.nextNode():e.nextNode()&&e.nextNode()}}},{key:"checkIframeFilter",value:function(e,t,n,r){var i=!1,o=!1;return r.forEach((function(e,t){e.val===n&&(i=t,o=e.handled)})),this.compareNodeIframe(e,t,n)?(!1!==i||o?!1===i||o||(r[i].handled=!0):r.push({val:n,handled:!0}),!0):(!1===i&&r.push({val:n,handled:!1}),!1)}},{key:"handleOpenIframes",value:function(e,t,n,r){var i=this;e.forEach((function(e){e.handled||i.getIframeContents(e.val,(function(e){i.createInstanceOnIframe(e).forEachNode(t,n,r)}))}))}},{key:"iterateThroughNodes",value:function(e,t,n,r,i){for(var o,a=this,s=this.createIterator(t,e,r),c=[],u=[],l=void 0,h=void 0;void 0,o=a.getIteratorNode(s),h=o.prevNode,l=o.node;)this.iframes&&this.forEachIframe(t,(function(e){return a.checkIframeFilter(l,h,e,c)}),(function(t){a.createInstanceOnIframe(t).forEachNode(e,(function(e){return u.push(e)}),r)})),u.push(l);u.forEach((function(e){n(e)})),this.iframes&&this.handleOpenIframes(c,e,n,r),i()}},{key:"forEachNode",value:function(e,t,n){var r=this,i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:function(){},o=this.getContexts(),a=o.length;a||i(),o.forEach((function(o){var s=function(){r.iterateThroughNodes(e,o,t,n,(function(){--a<=0&&i()}))};r.iframes?r.waitForIframes(o,s):s()}))}}],[{key:"matches",value:function(e,t){var n="string"==typeof t?[t]:t,r=e.matches||e.matchesSelector||e.msMatchesSelector||e.mozMatchesSelector||e.oMatchesSelector||e.webkitMatchesSelector;if(r){var i=!1;return n.every((function(t){return!r.call(e,t)||(i=!0,!1)})),i}return!1}}]),e}(),a=function(){function e(t){n(this,e),this.ctx=t,this.ie=!1;var r=window.navigator.userAgent;(r.indexOf("MSIE")>-1||r.indexOf("Trident")>-1)&&(this.ie=!0)}return r(e,[{key:"log",value:function(e){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"debug",r=this.opt.log;this.opt.debug&&"object"===(void 0===r?"undefined":t(r))&&"function"==typeof r[n]&&r[n]("mark.js: "+e)}},{key:"escapeStr",value:function(e){return e.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g,"\\$&")}},{key:"createRegExp",value:function(e){return"disabled"!==this.opt.wildcards&&(e=this.setupWildcardsRegExp(e)),e=this.escapeStr(e),Object.keys(this.opt.synonyms).length&&(e=this.createSynonymsRegExp(e)),(this.opt.ignoreJoiners||this.opt.ignorePunctuation.length)&&(e=this.setupIgnoreJoinersRegExp(e)),this.opt.diacritics&&(e=this.createDiacriticsRegExp(e)),e=this.createMergedBlanksRegExp(e),(this.opt.ignoreJoiners||this.opt.ignorePunctuation.length)&&(e=this.createJoinersRegExp(e)),"disabled"!==this.opt.wildcards&&(e=this.createWildcardsRegExp(e)),e=this.createAccuracyRegExp(e)}},{key:"createSynonymsRegExp",value:function(e){var t=this.opt.synonyms,n=this.opt.caseSensitive?"":"i",r=this.opt.ignoreJoiners||this.opt.ignorePunctuation.length?"\0":"";for(var i in t)if(t.hasOwnProperty(i)){var o=t[i],a="disabled"!==this.opt.wildcards?this.setupWildcardsRegExp(i):this.escapeStr(i),s="disabled"!==this.opt.wildcards?this.setupWildcardsRegExp(o):this.escapeStr(o);""!==a&&""!==s&&(e=e.replace(new RegExp("("+this.escapeStr(a)+"|"+this.escapeStr(s)+")","gm"+n),r+"("+this.processSynomyms(a)+"|"+this.processSynomyms(s)+")"+r))}return e}},{key:"processSynomyms",value:function(e){return(this.opt.ignoreJoiners||this.opt.ignorePunctuation.length)&&(e=this.setupIgnoreJoinersRegExp(e)),e}},{key:"setupWildcardsRegExp",value:function(e){return(e=e.replace(/(?:\\)*\?/g,(function(e){return"\\"===e.charAt(0)?"?":""}))).replace(/(?:\\)*\*/g,(function(e){return"\\"===e.charAt(0)?"*":""}))}},{key:"createWildcardsRegExp",value:function(e){var t="withSpaces"===this.opt.wildcards;return e.replace(/\u0001/g,t?"[\\S\\s]?":"\\S?").replace(/\u0002/g,t?"[\\S\\s]*?":"\\S*")}},{key:"setupIgnoreJoinersRegExp",value:function(e){return e.replace(/[^(|)\\]/g,(function(e,t,n){var r=n.charAt(t+1);return/[(|)\\]/.test(r)||""===r?e:e+"\0"}))}},{key:"createJoinersRegExp",value:function(e){var t=[],n=this.opt.ignorePunctuation;return Array.isArray(n)&&n.length&&t.push(this.escapeStr(n.join(""))),this.opt.ignoreJoiners&&t.push("\\u00ad\\u200b\\u200c\\u200d"),t.length?e.split(/\u0000+/).join("["+t.join("")+"]*"):e}},{key:"createDiacriticsRegExp",value:function(e){var t=this.opt.caseSensitive?"":"i",n=this.opt.caseSensitive?["aàáảãạăằắẳẵặâầấẩẫậäåÄÄ…","AÀÃẢÃẠĂẰẮẲẴẶÂẦẤẨẪẬÄÅĀĄ","cçćÄ","CÇĆČ","dÄ‘Ä","DÄÄŽ","eèéẻẽẹêá»áº¿á»ƒá»…ệëěēę","EÈÉẺẼẸÊỀẾỂỄỆËĚĒĘ","iìíỉĩịîïī","IÃŒÃỈĨỊÎÃĪ","lÅ‚","LÅ","nñňń","NÑŇŃ","oòóá»Ãµá»Ã´á»“ốổỗộơởỡớá»á»£Ã¶Ã¸Å","OÒÓỎÕỌÔỒá»á»”ỖỘƠỞỠỚỜỢÖØŌ","rÅ™","RŘ","sšśșş","SŠŚȘŞ","tťțţ","TŤȚŢ","uùúủũụưừứửữựûüůū","UÙÚỦŨỤƯỪỨỬỮỰÛÜŮŪ","yýỳỷỹỵÿ","YÃỲỶỸỴŸ","zžżź","ZŽŻŹ"]:["aàáảãạăằắẳẵặâầấẩẫậäåÄÄ…AÀÃẢÃẠĂẰẮẲẴẶÂẦẤẨẪẬÄÅĀĄ","cçćÄCÇĆČ","dÄ‘ÄDÄÄŽ","eèéẻẽẹêá»áº¿á»ƒá»…ệëěēęEÈÉẺẼẸÊỀẾỂỄỆËĚĒĘ","iìíỉĩịîïīIÃŒÃỈĨỊÎÃĪ","lÅ‚LÅ","nñňńNÑŇŃ","oòóá»Ãµá»Ã´á»“ốổỗộơởỡớá»á»£Ã¶Ã¸ÅOÒÓỎÕỌÔỒá»á»”ỖỘƠỞỠỚỜỢÖØŌ","rÅ™RŘ","sšśșşSŠŚȘŞ","tťțţTŤȚŢ","uùúủũụưừứửữựûüůūUÙÚỦŨỤƯỪỨỬỮỰÛÜŮŪ","yýỳỷỹỵÿYÃỲỶỸỴŸ","zžżźZŽŻŹ"],r=[];return e.split("").forEach((function(i){n.every((function(n){if(-1!==n.indexOf(i)){if(r.indexOf(n)>-1)return!1;e=e.replace(new RegExp("["+n+"]","gm"+t),"["+n+"]"),r.push(n)}return!0}))})),e}},{key:"createMergedBlanksRegExp",value:function(e){return e.replace(/[\s]+/gim,"[\\s]+")}},{key:"createAccuracyRegExp",value:function(e){var t=this,n=this.opt.accuracy,r="string"==typeof n?n:n.value,i="";switch(("string"==typeof n?[]:n.limiters).forEach((function(e){i+="|"+t.escapeStr(e)})),r){case"partially":default:return"()("+e+")";case"complementary":return"()([^"+(i="\\s"+(i||this.escapeStr("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~¡¿")))+"]*"+e+"[^"+i+"]*)";case"exactly":return"(^|\\s"+i+")("+e+")(?=$|\\s"+i+")"}}},{key:"getSeparatedKeywords",value:function(e){var t=this,n=[];return e.forEach((function(e){t.opt.separateWordSearch?e.split(" ").forEach((function(e){e.trim()&&-1===n.indexOf(e)&&n.push(e)})):e.trim()&&-1===n.indexOf(e)&&n.push(e)})),{keywords:n.sort((function(e,t){return t.length-e.length})),length:n.length}}},{key:"isNumeric",value:function(e){return Number(parseFloat(e))==e}},{key:"checkRanges",value:function(e){var t=this;if(!Array.isArray(e)||"[object Object]"!==Object.prototype.toString.call(e[0]))return this.log("markRanges() will only accept an array of objects"),this.opt.noMatch(e),[];var n=[],r=0;return e.sort((function(e,t){return e.start-t.start})).forEach((function(e){var i=t.callNoMatchOnInvalidRanges(e,r),o=i.start,a=i.end;i.valid&&(e.start=o,e.length=a-o,n.push(e),r=a)})),n}},{key:"callNoMatchOnInvalidRanges",value:function(e,t){var n=void 0,r=void 0,i=!1;return e&&void 0!==e.start?(r=(n=parseInt(e.start,10))+parseInt(e.length,10),this.isNumeric(e.start)&&this.isNumeric(e.length)&&r-t>0&&r-n>0?i=!0:(this.log("Ignoring invalid or overlapping range: "+JSON.stringify(e)),this.opt.noMatch(e))):(this.log("Ignoring invalid range: "+JSON.stringify(e)),this.opt.noMatch(e)),{start:n,end:r,valid:i}}},{key:"checkWhitespaceRanges",value:function(e,t,n){var r=void 0,i=!0,o=n.length,a=t-o,s=parseInt(e.start,10)-a;return(r=(s=s>o?o:s)+parseInt(e.length,10))>o&&(r=o,this.log("End range automatically set to the max value of "+o)),s<0||r-s<0||s>o||r>o?(i=!1,this.log("Invalid range: "+JSON.stringify(e)),this.opt.noMatch(e)):""===n.substring(s,r).replace(/\s+/g,"")&&(i=!1,this.log("Skipping whitespace only range: "+JSON.stringify(e)),this.opt.noMatch(e)),{start:s,end:r,valid:i}}},{key:"getTextNodes",value:function(e){var t=this,n="",r=[];this.iterator.forEachNode(NodeFilter.SHOW_TEXT,(function(e){r.push({start:n.length,end:(n+=e.textContent).length,node:e})}),(function(e){return t.matchesExclude(e.parentNode)?NodeFilter.FILTER_REJECT:NodeFilter.FILTER_ACCEPT}),(function(){e({value:n,nodes:r})}))}},{key:"matchesExclude",value:function(e){return o.matches(e,this.opt.exclude.concat(["script","style","title","head","html"]))}},{key:"wrapRangeInTextNode",value:function(e,t,n){var r=this.opt.element?this.opt.element:"mark",i=e.splitText(t),o=i.splitText(n-t),a=document.createElement(r);return a.setAttribute("data-markjs","true"),this.opt.className&&a.setAttribute("class",this.opt.className),a.textContent=i.textContent,i.parentNode.replaceChild(a,i),o}},{key:"wrapRangeInMappedTextNode",value:function(e,t,n,r,i){var o=this;e.nodes.every((function(a,s){var c=e.nodes[s+1];if(void 0===c||c.start>t){if(!r(a.node))return!1;var u=t-a.start,l=(n>a.end?a.end:n)-a.start,h=e.value.substr(0,a.start),f=e.value.substr(l+a.start);if(a.node=o.wrapRangeInTextNode(a.node,u,l),e.value=h+f,e.nodes.forEach((function(t,n){n>=s&&(e.nodes[n].start>0&&n!==s&&(e.nodes[n].start-=l),e.nodes[n].end-=l)})),n-=l,i(a.node.previousSibling,a.start),!(n>a.end))return!1;t=a.end}return!0}))}},{key:"wrapMatches",value:function(e,t,n,r,i){var o=this,a=0===t?0:t+1;this.getTextNodes((function(t){t.nodes.forEach((function(t){t=t.node;for(var i=void 0;null!==(i=e.exec(t.textContent))&&""!==i[a];)if(n(i[a],t)){var s=i.index;if(0!==a)for(var c=1;c.anchorjs-link,.anchorjs-link:focus{opacity:1}",u.sheet.cssRules.length),u.sheet.insertRule("[data-anchorjs-icon]::after{content:attr(data-anchorjs-icon)}",u.sheet.cssRules.length),u.sheet.insertRule('@font-face{font-family:anchorjs-icons;src:url(data:n/a;base64,AAEAAAALAIAAAwAwT1MvMg8yG2cAAAE4AAAAYGNtYXDp3gC3AAABpAAAAExnYXNwAAAAEAAAA9wAAAAIZ2x5ZlQCcfwAAAH4AAABCGhlYWQHFvHyAAAAvAAAADZoaGVhBnACFwAAAPQAAAAkaG10eASAADEAAAGYAAAADGxvY2EACACEAAAB8AAAAAhtYXhwAAYAVwAAARgAAAAgbmFtZQGOH9cAAAMAAAAAunBvc3QAAwAAAAADvAAAACAAAQAAAAEAAHzE2p9fDzz1AAkEAAAAAADRecUWAAAAANQA6R8AAAAAAoACwAAAAAgAAgAAAAAAAAABAAADwP/AAAACgAAA/9MCrQABAAAAAAAAAAAAAAAAAAAAAwABAAAAAwBVAAIAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAMCQAGQAAUAAAKZAswAAACPApkCzAAAAesAMwEJAAAAAAAAAAAAAAAAAAAAARAAAAAAAAAAAAAAAAAAAAAAQAAg//0DwP/AAEADwABAAAAAAQAAAAAAAAAAAAAAIAAAAAAAAAIAAAACgAAxAAAAAwAAAAMAAAAcAAEAAwAAABwAAwABAAAAHAAEADAAAAAIAAgAAgAAACDpy//9//8AAAAg6cv//f///+EWNwADAAEAAAAAAAAAAAAAAAAACACEAAEAAAAAAAAAAAAAAAAxAAACAAQARAKAAsAAKwBUAAABIiYnJjQ3NzY2MzIWFxYUBwcGIicmNDc3NjQnJiYjIgYHBwYUFxYUBwYGIwciJicmNDc3NjIXFhQHBwYUFxYWMzI2Nzc2NCcmNDc2MhcWFAcHBgYjARQGDAUtLXoWOR8fORYtLTgKGwoKCjgaGg0gEhIgDXoaGgkJBQwHdR85Fi0tOAobCgoKOBoaDSASEiANehoaCQkKGwotLXoWOR8BMwUFLYEuehYXFxYugC44CQkKGwo4GkoaDQ0NDXoaShoKGwoFBe8XFi6ALjgJCQobCjgaShoNDQ0NehpKGgobCgoKLYEuehYXAAAADACWAAEAAAAAAAEACAAAAAEAAAAAAAIAAwAIAAEAAAAAAAMACAAAAAEAAAAAAAQACAAAAAEAAAAAAAUAAQALAAEAAAAAAAYACAAAAAMAAQQJAAEAEAAMAAMAAQQJAAIABgAcAAMAAQQJAAMAEAAMAAMAAQQJAAQAEAAMAAMAAQQJAAUAAgAiAAMAAQQJAAYAEAAMYW5jaG9yanM0MDBAAGEAbgBjAGgAbwByAGoAcwA0ADAAMABAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAP) format("truetype")}',u.sheet.cssRules.length)),u=document.querySelectorAll("[id]"),t=[].map.call(u,(function(A){return A.id})),i=0;i\]./()*\\\n\t\b\v\u00A0]/g,"-").replace(/-{2,}/g,"-").substring(0,this.options.truncate).replace(/^-+|-+$/gm,"").toLowerCase()},this.hasAnchorJSLink=function(A){var e=A.firstChild&&-1<(" "+A.firstChild.className+" ").indexOf(" anchorjs-link "),A=A.lastChild&&-1<(" "+A.lastChild.className+" ").indexOf(" anchorjs-link ");return e||A||!1}}})); diff --git a/csharp/ApiDocs/_exported_templates/default/styles/lunr.js b/csharp/ApiDocs/_exported_templates/default/styles/lunr.js new file mode 100644 index 0000000000..8f2e2491aa --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/lunr.js @@ -0,0 +1,2924 @@ +/** + * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.1.2 + * Copyright (C) 2017 Oliver Nightingale + * @license MIT + */ + +;(function(){ + +/** + * A convenience function for configuring and constructing + * a new lunr Index. + * + * A lunr.Builder instance is created and the pipeline setup + * with a trimmer, stop word filter and stemmer. + * + * This builder object is yielded to the configuration function + * that is passed as a parameter, allowing the list of fields + * and other builder parameters to be customized. + * + * All documents _must_ be added within the passed config function. + * + * @example + * var idx = lunr(function () { + * this.field('title') + * this.field('body') + * this.ref('id') + * + * documents.forEach(function (doc) { + * this.add(doc) + * }, this) + * }) + * + * @see {@link lunr.Builder} + * @see {@link lunr.Pipeline} + * @see {@link lunr.trimmer} + * @see {@link lunr.stopWordFilter} + * @see {@link lunr.stemmer} + * @namespace {function} lunr + */ +var lunr = function (config) { + var builder = new lunr.Builder + + builder.pipeline.add( + lunr.trimmer, + lunr.stopWordFilter, + lunr.stemmer + ) + + builder.searchPipeline.add( + lunr.stemmer + ) + + config.call(builder, builder) + return builder.build() +} + +lunr.version = "2.1.2" +/*! + * lunr.utils + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * A namespace containing utils for the rest of the lunr library + */ +lunr.utils = {} + +/** + * Print a warning message to the console. + * + * @param {String} message The message to be printed. + * @memberOf Utils + */ +lunr.utils.warn = (function (global) { + /* eslint-disable no-console */ + return function (message) { + if (global.console && console.warn) { + console.warn(message) + } + } + /* eslint-enable no-console */ +})(this) + +/** + * Convert an object to a string. + * + * In the case of `null` and `undefined` the function returns + * the empty string, in all other cases the result of calling + * `toString` on the passed object is returned. + * + * @param {Any} obj The object to convert to a string. + * @return {String} string representation of the passed object. + * @memberOf Utils + */ +lunr.utils.asString = function (obj) { + if (obj === void 0 || obj === null) { + return "" + } else { + return obj.toString() + } +} +lunr.FieldRef = function (docRef, fieldName) { + this.docRef = docRef + this.fieldName = fieldName + this._stringValue = fieldName + lunr.FieldRef.joiner + docRef +} + +lunr.FieldRef.joiner = "/" + +lunr.FieldRef.fromString = function (s) { + var n = s.indexOf(lunr.FieldRef.joiner) + + if (n === -1) { + throw "malformed field ref string" + } + + var fieldRef = s.slice(0, n), + docRef = s.slice(n + 1) + + return new lunr.FieldRef (docRef, fieldRef) +} + +lunr.FieldRef.prototype.toString = function () { + return this._stringValue +} +/** + * A function to calculate the inverse document frequency for + * a posting. This is shared between the builder and the index + * + * @private + * @param {object} posting - The posting for a given term + * @param {number} documentCount - The total number of documents. + */ +lunr.idf = function (posting, documentCount) { + var documentsWithTerm = 0 + + for (var fieldName in posting) { + if (fieldName == '_index') continue // Ignore the term index, its not a field + documentsWithTerm += Object.keys(posting[fieldName]).length + } + + var x = (documentCount - documentsWithTerm + 0.5) / (documentsWithTerm + 0.5) + + return Math.log(1 + Math.abs(x)) +} + +/** + * A token wraps a string representation of a token + * as it is passed through the text processing pipeline. + * + * @constructor + * @param {string} [str=''] - The string token being wrapped. + * @param {object} [metadata={}] - Metadata associated with this token. + */ +lunr.Token = function (str, metadata) { + this.str = str || "" + this.metadata = metadata || {} +} + +/** + * Returns the token string that is being wrapped by this object. + * + * @returns {string} + */ +lunr.Token.prototype.toString = function () { + return this.str +} + +/** + * A token update function is used when updating or optionally + * when cloning a token. + * + * @callback lunr.Token~updateFunction + * @param {string} str - The string representation of the token. + * @param {Object} metadata - All metadata associated with this token. + */ + +/** + * Applies the given function to the wrapped string token. + * + * @example + * token.update(function (str, metadata) { + * return str.toUpperCase() + * }) + * + * @param {lunr.Token~updateFunction} fn - A function to apply to the token string. + * @returns {lunr.Token} + */ +lunr.Token.prototype.update = function (fn) { + this.str = fn(this.str, this.metadata) + return this +} + +/** + * Creates a clone of this token. Optionally a function can be + * applied to the cloned token. + * + * @param {lunr.Token~updateFunction} [fn] - An optional function to apply to the cloned token. + * @returns {lunr.Token} + */ +lunr.Token.prototype.clone = function (fn) { + fn = fn || function (s) { return s } + return new lunr.Token (fn(this.str, this.metadata), this.metadata) +} +/*! + * lunr.tokenizer + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * A function for splitting a string into tokens ready to be inserted into + * the search index. Uses `lunr.tokenizer.separator` to split strings, change + * the value of this property to change how strings are split into tokens. + * + * This tokenizer will convert its parameter to a string by calling `toString` and + * then will split this string on the character in `lunr.tokenizer.separator`. + * Arrays will have their elements converted to strings and wrapped in a lunr.Token. + * + * @static + * @param {?(string|object|object[])} obj - The object to convert into tokens + * @returns {lunr.Token[]} + */ +lunr.tokenizer = function (obj) { + if (obj == null || obj == undefined) { + return [] + } + + if (Array.isArray(obj)) { + return obj.map(function (t) { + return new lunr.Token(lunr.utils.asString(t).toLowerCase()) + }) + } + + var str = obj.toString().trim().toLowerCase(), + len = str.length, + tokens = [] + + for (var sliceEnd = 0, sliceStart = 0; sliceEnd <= len; sliceEnd++) { + var char = str.charAt(sliceEnd), + sliceLength = sliceEnd - sliceStart + + if ((char.match(lunr.tokenizer.separator) || sliceEnd == len)) { + + if (sliceLength > 0) { + tokens.push( + new lunr.Token (str.slice(sliceStart, sliceEnd), { + position: [sliceStart, sliceLength], + index: tokens.length + }) + ) + } + + sliceStart = sliceEnd + 1 + } + + } + + return tokens +} + +/** + * The separator used to split a string into tokens. Override this property to change the behavior of + * `lunr.tokenizer` behavior when tokenizing strings. By default this splits on whitespace and hyphens. + * + * @static + * @see lunr.tokenizer + */ +lunr.tokenizer.separator = /[\s\-]+/ +/*! + * lunr.Pipeline + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * lunr.Pipelines maintain an ordered list of functions to be applied to all + * tokens in documents entering the search index and queries being ran against + * the index. + * + * An instance of lunr.Index created with the lunr shortcut will contain a + * pipeline with a stop word filter and an English language stemmer. Extra + * functions can be added before or after either of these functions or these + * default functions can be removed. + * + * When run the pipeline will call each function in turn, passing a token, the + * index of that token in the original list of all tokens and finally a list of + * all the original tokens. + * + * The output of functions in the pipeline will be passed to the next function + * in the pipeline. To exclude a token from entering the index the function + * should return undefined, the rest of the pipeline will not be called with + * this token. + * + * For serialization of pipelines to work, all functions used in an instance of + * a pipeline should be registered with lunr.Pipeline. Registered functions can + * then be loaded. If trying to load a serialized pipeline that uses functions + * that are not registered an error will be thrown. + * + * If not planning on serializing the pipeline then registering pipeline functions + * is not necessary. + * + * @constructor + */ +lunr.Pipeline = function () { + this._stack = [] +} + +lunr.Pipeline.registeredFunctions = Object.create(null) + +/** + * A pipeline function maps lunr.Token to lunr.Token. A lunr.Token contains the token + * string as well as all known metadata. A pipeline function can mutate the token string + * or mutate (or add) metadata for a given token. + * + * A pipeline function can indicate that the passed token should be discarded by returning + * null. This token will not be passed to any downstream pipeline functions and will not be + * added to the index. + * + * Multiple tokens can be returned by returning an array of tokens. Each token will be passed + * to any downstream pipeline functions and all will returned tokens will be added to the index. + * + * Any number of pipeline functions may be chained together using a lunr.Pipeline. + * + * @interface lunr.PipelineFunction + * @param {lunr.Token} token - A token from the document being processed. + * @param {number} i - The index of this token in the complete list of tokens for this document/field. + * @param {lunr.Token[]} tokens - All tokens for this document/field. + * @returns {(?lunr.Token|lunr.Token[])} + */ + +/** + * Register a function with the pipeline. + * + * Functions that are used in the pipeline should be registered if the pipeline + * needs to be serialized, or a serialized pipeline needs to be loaded. + * + * Registering a function does not add it to a pipeline, functions must still be + * added to instances of the pipeline for them to be used when running a pipeline. + * + * @param {lunr.PipelineFunction} fn - The function to check for. + * @param {String} label - The label to register this function with + */ +lunr.Pipeline.registerFunction = function (fn, label) { + if (label in this.registeredFunctions) { + lunr.utils.warn('Overwriting existing registered function: ' + label) + } + + fn.label = label + lunr.Pipeline.registeredFunctions[fn.label] = fn +} + +/** + * Warns if the function is not registered as a Pipeline function. + * + * @param {lunr.PipelineFunction} fn - The function to check for. + * @private + */ +lunr.Pipeline.warnIfFunctionNotRegistered = function (fn) { + var isRegistered = fn.label && (fn.label in this.registeredFunctions) + + if (!isRegistered) { + lunr.utils.warn('Function is not registered with pipeline. This may cause problems when serializing the index.\n', fn) + } +} + +/** + * Loads a previously serialized pipeline. + * + * All functions to be loaded must already be registered with lunr.Pipeline. + * If any function from the serialized data has not been registered then an + * error will be thrown. + * + * @param {Object} serialized - The serialized pipeline to load. + * @returns {lunr.Pipeline} + */ +lunr.Pipeline.load = function (serialized) { + var pipeline = new lunr.Pipeline + + serialized.forEach(function (fnName) { + var fn = lunr.Pipeline.registeredFunctions[fnName] + + if (fn) { + pipeline.add(fn) + } else { + throw new Error('Cannot load unregistered function: ' + fnName) + } + }) + + return pipeline +} + +/** + * Adds new functions to the end of the pipeline. + * + * Logs a warning if the function has not been registered. + * + * @param {lunr.PipelineFunction[]} functions - Any number of functions to add to the pipeline. + */ +lunr.Pipeline.prototype.add = function () { + var fns = Array.prototype.slice.call(arguments) + + fns.forEach(function (fn) { + lunr.Pipeline.warnIfFunctionNotRegistered(fn) + this._stack.push(fn) + }, this) +} + +/** + * Adds a single function after a function that already exists in the + * pipeline. + * + * Logs a warning if the function has not been registered. + * + * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline. + * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline. + */ +lunr.Pipeline.prototype.after = function (existingFn, newFn) { + lunr.Pipeline.warnIfFunctionNotRegistered(newFn) + + var pos = this._stack.indexOf(existingFn) + if (pos == -1) { + throw new Error('Cannot find existingFn') + } + + pos = pos + 1 + this._stack.splice(pos, 0, newFn) +} + +/** + * Adds a single function before a function that already exists in the + * pipeline. + * + * Logs a warning if the function has not been registered. + * + * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline. + * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline. + */ +lunr.Pipeline.prototype.before = function (existingFn, newFn) { + lunr.Pipeline.warnIfFunctionNotRegistered(newFn) + + var pos = this._stack.indexOf(existingFn) + if (pos == -1) { + throw new Error('Cannot find existingFn') + } + + this._stack.splice(pos, 0, newFn) +} + +/** + * Removes a function from the pipeline. + * + * @param {lunr.PipelineFunction} fn The function to remove from the pipeline. + */ +lunr.Pipeline.prototype.remove = function (fn) { + var pos = this._stack.indexOf(fn) + if (pos == -1) { + return + } + + this._stack.splice(pos, 1) +} + +/** + * Runs the current list of functions that make up the pipeline against the + * passed tokens. + * + * @param {Array} tokens The tokens to run through the pipeline. + * @returns {Array} + */ +lunr.Pipeline.prototype.run = function (tokens) { + var stackLength = this._stack.length + + for (var i = 0; i < stackLength; i++) { + var fn = this._stack[i] + + tokens = tokens.reduce(function (memo, token, j) { + var result = fn(token, j, tokens) + + if (result === void 0 || result === '') return memo + + return memo.concat(result) + }, []) + } + + return tokens +} + +/** + * Convenience method for passing a string through a pipeline and getting + * strings out. This method takes care of wrapping the passed string in a + * token and mapping the resulting tokens back to strings. + * + * @param {string} str - The string to pass through the pipeline. + * @returns {string[]} + */ +lunr.Pipeline.prototype.runString = function (str) { + var token = new lunr.Token (str) + + return this.run([token]).map(function (t) { + return t.toString() + }) +} + +/** + * Resets the pipeline by removing any existing processors. + * + */ +lunr.Pipeline.prototype.reset = function () { + this._stack = [] +} + +/** + * Returns a representation of the pipeline ready for serialization. + * + * Logs a warning if the function has not been registered. + * + * @returns {Array} + */ +lunr.Pipeline.prototype.toJSON = function () { + return this._stack.map(function (fn) { + lunr.Pipeline.warnIfFunctionNotRegistered(fn) + + return fn.label + }) +} +/*! + * lunr.Vector + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * A vector is used to construct the vector space of documents and queries. These + * vectors support operations to determine the similarity between two documents or + * a document and a query. + * + * Normally no parameters are required for initializing a vector, but in the case of + * loading a previously dumped vector the raw elements can be provided to the constructor. + * + * For performance reasons vectors are implemented with a flat array, where an elements + * index is immediately followed by its value. E.g. [index, value, index, value]. This + * allows the underlying array to be as sparse as possible and still offer decent + * performance when being used for vector calculations. + * + * @constructor + * @param {Number[]} [elements] - The flat list of element index and element value pairs. + */ +lunr.Vector = function (elements) { + this._magnitude = 0 + this.elements = elements || [] +} + + +/** + * Calculates the position within the vector to insert a given index. + * + * This is used internally by insert and upsert. If there are duplicate indexes then + * the position is returned as if the value for that index were to be updated, but it + * is the callers responsibility to check whether there is a duplicate at that index + * + * @param {Number} insertIdx - The index at which the element should be inserted. + * @returns {Number} + */ +lunr.Vector.prototype.positionForIndex = function (index) { + // For an empty vector the tuple can be inserted at the beginning + if (this.elements.length == 0) { + return 0 + } + + var start = 0, + end = this.elements.length / 2, + sliceLength = end - start, + pivotPoint = Math.floor(sliceLength / 2), + pivotIndex = this.elements[pivotPoint * 2] + + while (sliceLength > 1) { + if (pivotIndex < index) { + start = pivotPoint + } + + if (pivotIndex > index) { + end = pivotPoint + } + + if (pivotIndex == index) { + break + } + + sliceLength = end - start + pivotPoint = start + Math.floor(sliceLength / 2) + pivotIndex = this.elements[pivotPoint * 2] + } + + if (pivotIndex == index) { + return pivotPoint * 2 + } + + if (pivotIndex > index) { + return pivotPoint * 2 + } + + if (pivotIndex < index) { + return (pivotPoint + 1) * 2 + } +} + +/** + * Inserts an element at an index within the vector. + * + * Does not allow duplicates, will throw an error if there is already an entry + * for this index. + * + * @param {Number} insertIdx - The index at which the element should be inserted. + * @param {Number} val - The value to be inserted into the vector. + */ +lunr.Vector.prototype.insert = function (insertIdx, val) { + this.upsert(insertIdx, val, function () { + throw "duplicate index" + }) +} + +/** + * Inserts or updates an existing index within the vector. + * + * @param {Number} insertIdx - The index at which the element should be inserted. + * @param {Number} val - The value to be inserted into the vector. + * @param {function} fn - A function that is called for updates, the existing value and the + * requested value are passed as arguments + */ +lunr.Vector.prototype.upsert = function (insertIdx, val, fn) { + this._magnitude = 0 + var position = this.positionForIndex(insertIdx) + + if (this.elements[position] == insertIdx) { + this.elements[position + 1] = fn(this.elements[position + 1], val) + } else { + this.elements.splice(position, 0, insertIdx, val) + } +} + +/** + * Calculates the magnitude of this vector. + * + * @returns {Number} + */ +lunr.Vector.prototype.magnitude = function () { + if (this._magnitude) return this._magnitude + + var sumOfSquares = 0, + elementsLength = this.elements.length + + for (var i = 1; i < elementsLength; i += 2) { + var val = this.elements[i] + sumOfSquares += val * val + } + + return this._magnitude = Math.sqrt(sumOfSquares) +} + +/** + * Calculates the dot product of this vector and another vector. + * + * @param {lunr.Vector} otherVector - The vector to compute the dot product with. + * @returns {Number} + */ +lunr.Vector.prototype.dot = function (otherVector) { + var dotProduct = 0, + a = this.elements, b = otherVector.elements, + aLen = a.length, bLen = b.length, + aVal = 0, bVal = 0, + i = 0, j = 0 + + while (i < aLen && j < bLen) { + aVal = a[i], bVal = b[j] + if (aVal < bVal) { + i += 2 + } else if (aVal > bVal) { + j += 2 + } else if (aVal == bVal) { + dotProduct += a[i + 1] * b[j + 1] + i += 2 + j += 2 + } + } + + return dotProduct +} + +/** + * Calculates the cosine similarity between this vector and another + * vector. + * + * @param {lunr.Vector} otherVector - The other vector to calculate the + * similarity with. + * @returns {Number} + */ +lunr.Vector.prototype.similarity = function (otherVector) { + return this.dot(otherVector) / (this.magnitude() * otherVector.magnitude()) +} + +/** + * Converts the vector to an array of the elements within the vector. + * + * @returns {Number[]} + */ +lunr.Vector.prototype.toArray = function () { + var output = new Array (this.elements.length / 2) + + for (var i = 1, j = 0; i < this.elements.length; i += 2, j++) { + output[j] = this.elements[i] + } + + return output +} + +/** + * A JSON serializable representation of the vector. + * + * @returns {Number[]} + */ +lunr.Vector.prototype.toJSON = function () { + return this.elements +} +/* eslint-disable */ +/*! + * lunr.stemmer + * Copyright (C) 2017 Oliver Nightingale + * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt + */ + +/** + * lunr.stemmer is an english language stemmer, this is a JavaScript + * implementation of the PorterStemmer taken from http://tartarus.org/~martin + * + * @static + * @implements {lunr.PipelineFunction} + * @param {lunr.Token} token - The string to stem + * @returns {lunr.Token} + * @see {@link lunr.Pipeline} + */ +lunr.stemmer = (function(){ + var step2list = { + "ational" : "ate", + "tional" : "tion", + "enci" : "ence", + "anci" : "ance", + "izer" : "ize", + "bli" : "ble", + "alli" : "al", + "entli" : "ent", + "eli" : "e", + "ousli" : "ous", + "ization" : "ize", + "ation" : "ate", + "ator" : "ate", + "alism" : "al", + "iveness" : "ive", + "fulness" : "ful", + "ousness" : "ous", + "aliti" : "al", + "iviti" : "ive", + "biliti" : "ble", + "logi" : "log" + }, + + step3list = { + "icate" : "ic", + "ative" : "", + "alize" : "al", + "iciti" : "ic", + "ical" : "ic", + "ful" : "", + "ness" : "" + }, + + c = "[^aeiou]", // consonant + v = "[aeiouy]", // vowel + C = c + "[^aeiouy]*", // consonant sequence + V = v + "[aeiou]*", // vowel sequence + + mgr0 = "^(" + C + ")?" + V + C, // [C]VC... is m>0 + meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$", // [C]VC[V] is m=1 + mgr1 = "^(" + C + ")?" + V + C + V + C, // [C]VCVC... is m>1 + s_v = "^(" + C + ")?" + v; // vowel in stem + + var re_mgr0 = new RegExp(mgr0); + var re_mgr1 = new RegExp(mgr1); + var re_meq1 = new RegExp(meq1); + var re_s_v = new RegExp(s_v); + + var re_1a = /^(.+?)(ss|i)es$/; + var re2_1a = /^(.+?)([^s])s$/; + var re_1b = /^(.+?)eed$/; + var re2_1b = /^(.+?)(ed|ing)$/; + var re_1b_2 = /.$/; + var re2_1b_2 = /(at|bl|iz)$/; + var re3_1b_2 = new RegExp("([^aeiouylsz])\\1$"); + var re4_1b_2 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + + var re_1c = /^(.+?[^aeiou])y$/; + var re_2 = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; + + var re_3 = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; + + var re_4 = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; + var re2_4 = /^(.+?)(s|t)(ion)$/; + + var re_5 = /^(.+?)e$/; + var re_5_1 = /ll$/; + var re3_5 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + + var porterStemmer = function porterStemmer(w) { + var stem, + suffix, + firstch, + re, + re2, + re3, + re4; + + if (w.length < 3) { return w; } + + firstch = w.substr(0,1); + if (firstch == "y") { + w = firstch.toUpperCase() + w.substr(1); + } + + // Step 1a + re = re_1a + re2 = re2_1a; + + if (re.test(w)) { w = w.replace(re,"$1$2"); } + else if (re2.test(w)) { w = w.replace(re2,"$1$2"); } + + // Step 1b + re = re_1b; + re2 = re2_1b; + if (re.test(w)) { + var fp = re.exec(w); + re = re_mgr0; + if (re.test(fp[1])) { + re = re_1b_2; + w = w.replace(re,""); + } + } else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1]; + re2 = re_s_v; + if (re2.test(stem)) { + w = stem; + re2 = re2_1b_2; + re3 = re3_1b_2; + re4 = re4_1b_2; + if (re2.test(w)) { w = w + "e"; } + else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,""); } + else if (re4.test(w)) { w = w + "e"; } + } + } + + // Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by -> by, say -> say) + re = re_1c; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + w = stem + "i"; + } + + // Step 2 + re = re_2; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = re_mgr0; + if (re.test(stem)) { + w = stem + step2list[suffix]; + } + } + + // Step 3 + re = re_3; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = re_mgr0; + if (re.test(stem)) { + w = stem + step3list[suffix]; + } + } + + // Step 4 + re = re_4; + re2 = re2_4; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = re_mgr1; + if (re.test(stem)) { + w = stem; + } + } else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1] + fp[2]; + re2 = re_mgr1; + if (re2.test(stem)) { + w = stem; + } + } + + // Step 5 + re = re_5; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = re_mgr1; + re2 = re_meq1; + re3 = re3_5; + if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) { + w = stem; + } + } + + re = re_5_1; + re2 = re_mgr1; + if (re.test(w) && re2.test(w)) { + re = re_1b_2; + w = w.replace(re,""); + } + + // and turn initial Y back to y + + if (firstch == "y") { + w = firstch.toLowerCase() + w.substr(1); + } + + return w; + }; + + return function (token) { + return token.update(porterStemmer); + } +})(); + +lunr.Pipeline.registerFunction(lunr.stemmer, 'stemmer') +/*! + * lunr.stopWordFilter + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * lunr.generateStopWordFilter builds a stopWordFilter function from the provided + * list of stop words. + * + * The built in lunr.stopWordFilter is built using this generator and can be used + * to generate custom stopWordFilters for applications or non English languages. + * + * @param {Array} token The token to pass through the filter + * @returns {lunr.PipelineFunction} + * @see lunr.Pipeline + * @see lunr.stopWordFilter + */ +lunr.generateStopWordFilter = function (stopWords) { + var words = stopWords.reduce(function (memo, stopWord) { + memo[stopWord] = stopWord + return memo + }, {}) + + return function (token) { + if (token && words[token.toString()] !== token.toString()) return token + } +} + +/** + * lunr.stopWordFilter is an English language stop word list filter, any words + * contained in the list will not be passed through the filter. + * + * This is intended to be used in the Pipeline. If the token does not pass the + * filter then undefined will be returned. + * + * @implements {lunr.PipelineFunction} + * @params {lunr.Token} token - A token to check for being a stop word. + * @returns {lunr.Token} + * @see {@link lunr.Pipeline} + */ +lunr.stopWordFilter = lunr.generateStopWordFilter([ + 'a', + 'able', + 'about', + 'across', + 'after', + 'all', + 'almost', + 'also', + 'am', + 'among', + 'an', + 'and', + 'any', + 'are', + 'as', + 'at', + 'be', + 'because', + 'been', + 'but', + 'by', + 'can', + 'cannot', + 'could', + 'dear', + 'did', + 'do', + 'does', + 'either', + 'else', + 'ever', + 'every', + 'for', + 'from', + 'get', + 'got', + 'had', + 'has', + 'have', + 'he', + 'her', + 'hers', + 'him', + 'his', + 'how', + 'however', + 'i', + 'if', + 'in', + 'into', + 'is', + 'it', + 'its', + 'just', + 'least', + 'let', + 'like', + 'likely', + 'may', + 'me', + 'might', + 'most', + 'must', + 'my', + 'neither', + 'no', + 'nor', + 'not', + 'of', + 'off', + 'often', + 'on', + 'only', + 'or', + 'other', + 'our', + 'own', + 'rather', + 'said', + 'say', + 'says', + 'she', + 'should', + 'since', + 'so', + 'some', + 'than', + 'that', + 'the', + 'their', + 'them', + 'then', + 'there', + 'these', + 'they', + 'this', + 'tis', + 'to', + 'too', + 'twas', + 'us', + 'wants', + 'was', + 'we', + 'were', + 'what', + 'when', + 'where', + 'which', + 'while', + 'who', + 'whom', + 'why', + 'will', + 'with', + 'would', + 'yet', + 'you', + 'your' +]) + +lunr.Pipeline.registerFunction(lunr.stopWordFilter, 'stopWordFilter') +/*! + * lunr.trimmer + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * lunr.trimmer is a pipeline function for trimming non word + * characters from the beginning and end of tokens before they + * enter the index. + * + * This implementation may not work correctly for non latin + * characters and should either be removed or adapted for use + * with languages with non-latin characters. + * + * @static + * @implements {lunr.PipelineFunction} + * @param {lunr.Token} token The token to pass through the filter + * @returns {lunr.Token} + * @see lunr.Pipeline + */ +lunr.trimmer = function (token) { + return token.update(function (s) { + return s.replace(/^\W+/, '').replace(/\W+$/, '') + }) +} + +lunr.Pipeline.registerFunction(lunr.trimmer, 'trimmer') +/*! + * lunr.TokenSet + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * A token set is used to store the unique list of all tokens + * within an index. Token sets are also used to represent an + * incoming query to the index, this query token set and index + * token set are then intersected to find which tokens to look + * up in the inverted index. + * + * A token set can hold multiple tokens, as in the case of the + * index token set, or it can hold a single token as in the + * case of a simple query token set. + * + * Additionally token sets are used to perform wildcard matching. + * Leading, contained and trailing wildcards are supported, and + * from this edit distance matching can also be provided. + * + * Token sets are implemented as a minimal finite state automata, + * where both common prefixes and suffixes are shared between tokens. + * This helps to reduce the space used for storing the token set. + * + * @constructor + */ +lunr.TokenSet = function () { + this.final = false + this.edges = {} + this.id = lunr.TokenSet._nextId + lunr.TokenSet._nextId += 1 +} + +/** + * Keeps track of the next, auto increment, identifier to assign + * to a new tokenSet. + * + * TokenSets require a unique identifier to be correctly minimized. + * + * @private + */ +lunr.TokenSet._nextId = 1 + +/** + * Creates a TokenSet instance from the given sorted array of words. + * + * @param {String[]} arr - A sorted array of strings to create the set from. + * @returns {lunr.TokenSet} + * @throws Will throw an error if the input array is not sorted. + */ +lunr.TokenSet.fromArray = function (arr) { + var builder = new lunr.TokenSet.Builder + + for (var i = 0, len = arr.length; i < len; i++) { + builder.insert(arr[i]) + } + + builder.finish() + return builder.root +} + +/** + * Creates a token set from a query clause. + * + * @private + * @param {Object} clause - A single clause from lunr.Query. + * @param {string} clause.term - The query clause term. + * @param {number} [clause.editDistance] - The optional edit distance for the term. + * @returns {lunr.TokenSet} + */ +lunr.TokenSet.fromClause = function (clause) { + if ('editDistance' in clause) { + return lunr.TokenSet.fromFuzzyString(clause.term, clause.editDistance) + } else { + return lunr.TokenSet.fromString(clause.term) + } +} + +/** + * Creates a token set representing a single string with a specified + * edit distance. + * + * Insertions, deletions, substitutions and transpositions are each + * treated as an edit distance of 1. + * + * Increasing the allowed edit distance will have a dramatic impact + * on the performance of both creating and intersecting these TokenSets. + * It is advised to keep the edit distance less than 3. + * + * @param {string} str - The string to create the token set from. + * @param {number} editDistance - The allowed edit distance to match. + * @returns {lunr.Vector} + */ +lunr.TokenSet.fromFuzzyString = function (str, editDistance) { + var root = new lunr.TokenSet + + var stack = [{ + node: root, + editsRemaining: editDistance, + str: str + }] + + while (stack.length) { + var frame = stack.pop() + + // no edit + if (frame.str.length > 0) { + var char = frame.str.charAt(0), + noEditNode + + if (char in frame.node.edges) { + noEditNode = frame.node.edges[char] + } else { + noEditNode = new lunr.TokenSet + frame.node.edges[char] = noEditNode + } + + if (frame.str.length == 1) { + noEditNode.final = true + } else { + stack.push({ + node: noEditNode, + editsRemaining: frame.editsRemaining, + str: frame.str.slice(1) + }) + } + } + + // deletion + // can only do a deletion if we have enough edits remaining + // and if there are characters left to delete in the string + if (frame.editsRemaining > 0 && frame.str.length > 1) { + var char = frame.str.charAt(1), + deletionNode + + if (char in frame.node.edges) { + deletionNode = frame.node.edges[char] + } else { + deletionNode = new lunr.TokenSet + frame.node.edges[char] = deletionNode + } + + if (frame.str.length <= 2) { + deletionNode.final = true + } else { + stack.push({ + node: deletionNode, + editsRemaining: frame.editsRemaining - 1, + str: frame.str.slice(2) + }) + } + } + + // deletion + // just removing the last character from the str + if (frame.editsRemaining > 0 && frame.str.length == 1) { + frame.node.final = true + } + + // substitution + // can only do a substitution if we have enough edits remaining + // and if there are characters left to substitute + if (frame.editsRemaining > 0 && frame.str.length >= 1) { + if ("*" in frame.node.edges) { + var substitutionNode = frame.node.edges["*"] + } else { + var substitutionNode = new lunr.TokenSet + frame.node.edges["*"] = substitutionNode + } + + if (frame.str.length == 1) { + substitutionNode.final = true + } else { + stack.push({ + node: substitutionNode, + editsRemaining: frame.editsRemaining - 1, + str: frame.str.slice(1) + }) + } + } + + // insertion + // can only do insertion if there are edits remaining + if (frame.editsRemaining > 0) { + if ("*" in frame.node.edges) { + var insertionNode = frame.node.edges["*"] + } else { + var insertionNode = new lunr.TokenSet + frame.node.edges["*"] = insertionNode + } + + if (frame.str.length == 0) { + insertionNode.final = true + } else { + stack.push({ + node: insertionNode, + editsRemaining: frame.editsRemaining - 1, + str: frame.str + }) + } + } + + // transposition + // can only do a transposition if there are edits remaining + // and there are enough characters to transpose + if (frame.editsRemaining > 0 && frame.str.length > 1) { + var charA = frame.str.charAt(0), + charB = frame.str.charAt(1), + transposeNode + + if (charB in frame.node.edges) { + transposeNode = frame.node.edges[charB] + } else { + transposeNode = new lunr.TokenSet + frame.node.edges[charB] = transposeNode + } + + if (frame.str.length == 1) { + transposeNode.final = true + } else { + stack.push({ + node: transposeNode, + editsRemaining: frame.editsRemaining - 1, + str: charA + frame.str.slice(2) + }) + } + } + } + + return root +} + +/** + * Creates a TokenSet from a string. + * + * The string may contain one or more wildcard characters (*) + * that will allow wildcard matching when intersecting with + * another TokenSet. + * + * @param {string} str - The string to create a TokenSet from. + * @returns {lunr.TokenSet} + */ +lunr.TokenSet.fromString = function (str) { + var node = new lunr.TokenSet, + root = node, + wildcardFound = false + + /* + * Iterates through all characters within the passed string + * appending a node for each character. + * + * As soon as a wildcard character is found then a self + * referencing edge is introduced to continually match + * any number of any characters. + */ + for (var i = 0, len = str.length; i < len; i++) { + var char = str[i], + final = (i == len - 1) + + if (char == "*") { + wildcardFound = true + node.edges[char] = node + node.final = final + + } else { + var next = new lunr.TokenSet + next.final = final + + node.edges[char] = next + node = next + + // TODO: is this needed anymore? + if (wildcardFound) { + node.edges["*"] = root + } + } + } + + return root +} + +/** + * Converts this TokenSet into an array of strings + * contained within the TokenSet. + * + * @returns {string[]} + */ +lunr.TokenSet.prototype.toArray = function () { + var words = [] + + var stack = [{ + prefix: "", + node: this + }] + + while (stack.length) { + var frame = stack.pop(), + edges = Object.keys(frame.node.edges), + len = edges.length + + if (frame.node.final) { + words.push(frame.prefix) + } + + for (var i = 0; i < len; i++) { + var edge = edges[i] + + stack.push({ + prefix: frame.prefix.concat(edge), + node: frame.node.edges[edge] + }) + } + } + + return words +} + +/** + * Generates a string representation of a TokenSet. + * + * This is intended to allow TokenSets to be used as keys + * in objects, largely to aid the construction and minimisation + * of a TokenSet. As such it is not designed to be a human + * friendly representation of the TokenSet. + * + * @returns {string} + */ +lunr.TokenSet.prototype.toString = function () { + // NOTE: Using Object.keys here as this.edges is very likely + // to enter 'hash-mode' with many keys being added + // + // avoiding a for-in loop here as it leads to the function + // being de-optimized (at least in V8). From some simple + // benchmarks the performance is comparable, but allowing + // V8 to optimize may mean easy performance wins in the future. + + if (this._str) { + return this._str + } + + var str = this.final ? '1' : '0', + labels = Object.keys(this.edges).sort(), + len = labels.length + + for (var i = 0; i < len; i++) { + var label = labels[i], + node = this.edges[label] + + str = str + label + node.id + } + + return str +} + +/** + * Returns a new TokenSet that is the intersection of + * this TokenSet and the passed TokenSet. + * + * This intersection will take into account any wildcards + * contained within the TokenSet. + * + * @param {lunr.TokenSet} b - An other TokenSet to intersect with. + * @returns {lunr.TokenSet} + */ +lunr.TokenSet.prototype.intersect = function (b) { + var output = new lunr.TokenSet, + frame = undefined + + var stack = [{ + qNode: b, + output: output, + node: this + }] + + while (stack.length) { + frame = stack.pop() + + // NOTE: As with the #toString method, we are using + // Object.keys and a for loop instead of a for-in loop + // as both of these objects enter 'hash' mode, causing + // the function to be de-optimized in V8 + var qEdges = Object.keys(frame.qNode.edges), + qLen = qEdges.length, + nEdges = Object.keys(frame.node.edges), + nLen = nEdges.length + + for (var q = 0; q < qLen; q++) { + var qEdge = qEdges[q] + + for (var n = 0; n < nLen; n++) { + var nEdge = nEdges[n] + + if (nEdge == qEdge || qEdge == '*') { + var node = frame.node.edges[nEdge], + qNode = frame.qNode.edges[qEdge], + final = node.final && qNode.final, + next = undefined + + if (nEdge in frame.output.edges) { + // an edge already exists for this character + // no need to create a new node, just set the finality + // bit unless this node is already final + next = frame.output.edges[nEdge] + next.final = next.final || final + + } else { + // no edge exists yet, must create one + // set the finality bit and insert it + // into the output + next = new lunr.TokenSet + next.final = final + frame.output.edges[nEdge] = next + } + + stack.push({ + qNode: qNode, + output: next, + node: node + }) + } + } + } + } + + return output +} +lunr.TokenSet.Builder = function () { + this.previousWord = "" + this.root = new lunr.TokenSet + this.uncheckedNodes = [] + this.minimizedNodes = {} +} + +lunr.TokenSet.Builder.prototype.insert = function (word) { + var node, + commonPrefix = 0 + + if (word < this.previousWord) { + throw new Error ("Out of order word insertion") + } + + for (var i = 0; i < word.length && i < this.previousWord.length; i++) { + if (word[i] != this.previousWord[i]) break + commonPrefix++ + } + + this.minimize(commonPrefix) + + if (this.uncheckedNodes.length == 0) { + node = this.root + } else { + node = this.uncheckedNodes[this.uncheckedNodes.length - 1].child + } + + for (var i = commonPrefix; i < word.length; i++) { + var nextNode = new lunr.TokenSet, + char = word[i] + + node.edges[char] = nextNode + + this.uncheckedNodes.push({ + parent: node, + char: char, + child: nextNode + }) + + node = nextNode + } + + node.final = true + this.previousWord = word +} + +lunr.TokenSet.Builder.prototype.finish = function () { + this.minimize(0) +} + +lunr.TokenSet.Builder.prototype.minimize = function (downTo) { + for (var i = this.uncheckedNodes.length - 1; i >= downTo; i--) { + var node = this.uncheckedNodes[i], + childKey = node.child.toString() + + if (childKey in this.minimizedNodes) { + node.parent.edges[node.char] = this.minimizedNodes[childKey] + } else { + // Cache the key for this node since + // we know it can't change anymore + node.child._str = childKey + + this.minimizedNodes[childKey] = node.child + } + + this.uncheckedNodes.pop() + } +} +/*! + * lunr.Index + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * An index contains the built index of all documents and provides a query interface + * to the index. + * + * Usually instances of lunr.Index will not be created using this constructor, instead + * lunr.Builder should be used to construct new indexes, or lunr.Index.load should be + * used to load previously built and serialized indexes. + * + * @constructor + * @param {Object} attrs - The attributes of the built search index. + * @param {Object} attrs.invertedIndex - An index of term/field to document reference. + * @param {Object} attrs.documentVectors - Document vectors keyed by document reference. + * @param {lunr.TokenSet} attrs.tokenSet - An set of all corpus tokens. + * @param {string[]} attrs.fields - The names of indexed document fields. + * @param {lunr.Pipeline} attrs.pipeline - The pipeline to use for search terms. + */ +lunr.Index = function (attrs) { + this.invertedIndex = attrs.invertedIndex + this.fieldVectors = attrs.fieldVectors + this.tokenSet = attrs.tokenSet + this.fields = attrs.fields + this.pipeline = attrs.pipeline +} + +/** + * A result contains details of a document matching a search query. + * @typedef {Object} lunr.Index~Result + * @property {string} ref - The reference of the document this result represents. + * @property {number} score - A number between 0 and 1 representing how similar this document is to the query. + * @property {lunr.MatchData} matchData - Contains metadata about this match including which term(s) caused the match. + */ + +/** + * Although lunr provides the ability to create queries using lunr.Query, it also provides a simple + * query language which itself is parsed into an instance of lunr.Query. + * + * For programmatically building queries it is advised to directly use lunr.Query, the query language + * is best used for human entered text rather than program generated text. + * + * At its simplest queries can just be a single term, e.g. `hello`, multiple terms are also supported + * and will be combined with OR, e.g `hello world` will match documents that contain either 'hello' + * or 'world', though those that contain both will rank higher in the results. + * + * Wildcards can be included in terms to match one or more unspecified characters, these wildcards can + * be inserted anywhere within the term, and more than one wildcard can exist in a single term. Adding + * wildcards will increase the number of documents that will be found but can also have a negative + * impact on query performance, especially with wildcards at the beginning of a term. + * + * Terms can be restricted to specific fields, e.g. `title:hello`, only documents with the term + * hello in the title field will match this query. Using a field not present in the index will lead + * to an error being thrown. + * + * Modifiers can also be added to terms, lunr supports edit distance and boost modifiers on terms. A term + * boost will make documents matching that term score higher, e.g. `foo^5`. Edit distance is also supported + * to provide fuzzy matching, e.g. 'hello~2' will match documents with hello with an edit distance of 2. + * Avoid large values for edit distance to improve query performance. + * + * To escape special characters the backslash character '\' can be used, this allows searches to include + * characters that would normally be considered modifiers, e.g. `foo\~2` will search for a term "foo~2" instead + * of attempting to apply a boost of 2 to the search term "foo". + * + * @typedef {string} lunr.Index~QueryString + * @example Simple single term query + * hello + * @example Multiple term query + * hello world + * @example term scoped to a field + * title:hello + * @example term with a boost of 10 + * hello^10 + * @example term with an edit distance of 2 + * hello~2 + */ + +/** + * Performs a search against the index using lunr query syntax. + * + * Results will be returned sorted by their score, the most relevant results + * will be returned first. + * + * For more programmatic querying use lunr.Index#query. + * + * @param {lunr.Index~QueryString} queryString - A string containing a lunr query. + * @throws {lunr.QueryParseError} If the passed query string cannot be parsed. + * @returns {lunr.Index~Result[]} + */ +lunr.Index.prototype.search = function (queryString) { + return this.query(function (query) { + var parser = new lunr.QueryParser(queryString, query) + parser.parse() + }) +} + +/** + * A query builder callback provides a query object to be used to express + * the query to perform on the index. + * + * @callback lunr.Index~queryBuilder + * @param {lunr.Query} query - The query object to build up. + * @this lunr.Query + */ + +/** + * Performs a query against the index using the yielded lunr.Query object. + * + * If performing programmatic queries against the index, this method is preferred + * over lunr.Index#search so as to avoid the additional query parsing overhead. + * + * A query object is yielded to the supplied function which should be used to + * express the query to be run against the index. + * + * Note that although this function takes a callback parameter it is _not_ an + * asynchronous operation, the callback is just yielded a query object to be + * customized. + * + * @param {lunr.Index~queryBuilder} fn - A function that is used to build the query. + * @returns {lunr.Index~Result[]} + */ +lunr.Index.prototype.query = function (fn) { + // for each query clause + // * process terms + // * expand terms from token set + // * find matching documents and metadata + // * get document vectors + // * score documents + + var query = new lunr.Query(this.fields), + matchingFields = Object.create(null), + queryVectors = Object.create(null) + + fn.call(query, query) + + for (var i = 0; i < query.clauses.length; i++) { + /* + * Unless the pipeline has been disabled for this term, which is + * the case for terms with wildcards, we need to pass the clause + * term through the search pipeline. A pipeline returns an array + * of processed terms. Pipeline functions may expand the passed + * term, which means we may end up performing multiple index lookups + * for a single query term. + */ + var clause = query.clauses[i], + terms = null + + if (clause.usePipeline) { + terms = this.pipeline.runString(clause.term) + } else { + terms = [clause.term] + } + + for (var m = 0; m < terms.length; m++) { + var term = terms[m] + + /* + * Each term returned from the pipeline needs to use the same query + * clause object, e.g. the same boost and or edit distance. The + * simplest way to do this is to re-use the clause object but mutate + * its term property. + */ + clause.term = term + + /* + * From the term in the clause we create a token set which will then + * be used to intersect the indexes token set to get a list of terms + * to lookup in the inverted index + */ + var termTokenSet = lunr.TokenSet.fromClause(clause), + expandedTerms = this.tokenSet.intersect(termTokenSet).toArray() + + for (var j = 0; j < expandedTerms.length; j++) { + /* + * For each term get the posting and termIndex, this is required for + * building the query vector. + */ + var expandedTerm = expandedTerms[j], + posting = this.invertedIndex[expandedTerm], + termIndex = posting._index + + for (var k = 0; k < clause.fields.length; k++) { + /* + * For each field that this query term is scoped by (by default + * all fields are in scope) we need to get all the document refs + * that have this term in that field. + * + * The posting is the entry in the invertedIndex for the matching + * term from above. + */ + var field = clause.fields[k], + fieldPosting = posting[field], + matchingDocumentRefs = Object.keys(fieldPosting) + + /* + * To support field level boosts a query vector is created per + * field. This vector is populated using the termIndex found for + * the term and a unit value with the appropriate boost applied. + * + * If the query vector for this field does not exist yet it needs + * to be created. + */ + if (!(field in queryVectors)) { + queryVectors[field] = new lunr.Vector + } + + /* + * Using upsert because there could already be an entry in the vector + * for the term we are working with. In that case we just add the scores + * together. + */ + queryVectors[field].upsert(termIndex, 1 * clause.boost, function (a, b) { return a + b }) + + for (var l = 0; l < matchingDocumentRefs.length; l++) { + /* + * All metadata for this term/field/document triple + * are then extracted and collected into an instance + * of lunr.MatchData ready to be returned in the query + * results + */ + var matchingDocumentRef = matchingDocumentRefs[l], + matchingFieldRef = new lunr.FieldRef (matchingDocumentRef, field), + documentMetadata, matchData + + documentMetadata = fieldPosting[matchingDocumentRef] + matchData = new lunr.MatchData (expandedTerm, field, documentMetadata) + + if (matchingFieldRef in matchingFields) { + matchingFields[matchingFieldRef].combine(matchData) + } else { + matchingFields[matchingFieldRef] = matchData + } + + } + } + } + } + } + + var matchingFieldRefs = Object.keys(matchingFields), + results = {} + + for (var i = 0; i < matchingFieldRefs.length; i++) { + /* + * Currently we have document fields that match the query, but we + * need to return documents. The matchData and scores are combined + * from multiple fields belonging to the same document. + * + * Scores are calculated by field, using the query vectors created + * above, and combined into a final document score using addition. + */ + var fieldRef = lunr.FieldRef.fromString(matchingFieldRefs[i]), + docRef = fieldRef.docRef, + fieldVector = this.fieldVectors[fieldRef], + score = queryVectors[fieldRef.fieldName].similarity(fieldVector) + + if (docRef in results) { + results[docRef].score += score + results[docRef].matchData.combine(matchingFields[fieldRef]) + } else { + results[docRef] = { + ref: docRef, + score: score, + matchData: matchingFields[fieldRef] + } + } + } + + /* + * The results object needs to be converted into a list + * of results, sorted by score before being returned. + */ + return Object.keys(results) + .map(function (key) { + return results[key] + }) + .sort(function (a, b) { + return b.score - a.score + }) +} + +/** + * Prepares the index for JSON serialization. + * + * The schema for this JSON blob will be described in a + * separate JSON schema file. + * + * @returns {Object} + */ +lunr.Index.prototype.toJSON = function () { + var invertedIndex = Object.keys(this.invertedIndex) + .sort() + .map(function (term) { + return [term, this.invertedIndex[term]] + }, this) + + var fieldVectors = Object.keys(this.fieldVectors) + .map(function (ref) { + return [ref, this.fieldVectors[ref].toJSON()] + }, this) + + return { + version: lunr.version, + fields: this.fields, + fieldVectors: fieldVectors, + invertedIndex: invertedIndex, + pipeline: this.pipeline.toJSON() + } +} + +/** + * Loads a previously serialized lunr.Index + * + * @param {Object} serializedIndex - A previously serialized lunr.Index + * @returns {lunr.Index} + */ +lunr.Index.load = function (serializedIndex) { + var attrs = {}, + fieldVectors = {}, + serializedVectors = serializedIndex.fieldVectors, + invertedIndex = {}, + serializedInvertedIndex = serializedIndex.invertedIndex, + tokenSetBuilder = new lunr.TokenSet.Builder, + pipeline = lunr.Pipeline.load(serializedIndex.pipeline) + + if (serializedIndex.version != lunr.version) { + lunr.utils.warn("Version mismatch when loading serialized index. Current version of lunr '" + lunr.version + "' does not match serialized index '" + serializedIndex.version + "'") + } + + for (var i = 0; i < serializedVectors.length; i++) { + var tuple = serializedVectors[i], + ref = tuple[0], + elements = tuple[1] + + fieldVectors[ref] = new lunr.Vector(elements) + } + + for (var i = 0; i < serializedInvertedIndex.length; i++) { + var tuple = serializedInvertedIndex[i], + term = tuple[0], + posting = tuple[1] + + tokenSetBuilder.insert(term) + invertedIndex[term] = posting + } + + tokenSetBuilder.finish() + + attrs.fields = serializedIndex.fields + + attrs.fieldVectors = fieldVectors + attrs.invertedIndex = invertedIndex + attrs.tokenSet = tokenSetBuilder.root + attrs.pipeline = pipeline + + return new lunr.Index(attrs) +} +/*! + * lunr.Builder + * Copyright (C) 2017 Oliver Nightingale + */ + +/** + * lunr.Builder performs indexing on a set of documents and + * returns instances of lunr.Index ready for querying. + * + * All configuration of the index is done via the builder, the + * fields to index, the document reference, the text processing + * pipeline and document scoring parameters are all set on the + * builder before indexing. + * + * @constructor + * @property {string} _ref - Internal reference to the document reference field. + * @property {string[]} _fields - Internal reference to the document fields to index. + * @property {object} invertedIndex - The inverted index maps terms to document fields. + * @property {object} documentTermFrequencies - Keeps track of document term frequencies. + * @property {object} documentLengths - Keeps track of the length of documents added to the index. + * @property {lunr.tokenizer} tokenizer - Function for splitting strings into tokens for indexing. + * @property {lunr.Pipeline} pipeline - The pipeline performs text processing on tokens before indexing. + * @property {lunr.Pipeline} searchPipeline - A pipeline for processing search terms before querying the index. + * @property {number} documentCount - Keeps track of the total number of documents indexed. + * @property {number} _b - A parameter to control field length normalization, setting this to 0 disabled normalization, 1 fully normalizes field lengths, the default value is 0.75. + * @property {number} _k1 - A parameter to control how quickly an increase in term frequency results in term frequency saturation, the default value is 1.2. + * @property {number} termIndex - A counter incremented for each unique term, used to identify a terms position in the vector space. + * @property {array} metadataWhitelist - A list of metadata keys that have been whitelisted for entry in the index. + */ +lunr.Builder = function () { + this._ref = "id" + this._fields = [] + this.invertedIndex = Object.create(null) + this.fieldTermFrequencies = {} + this.fieldLengths = {} + this.tokenizer = lunr.tokenizer + this.pipeline = new lunr.Pipeline + this.searchPipeline = new lunr.Pipeline + this.documentCount = 0 + this._b = 0.75 + this._k1 = 1.2 + this.termIndex = 0 + this.metadataWhitelist = [] +} + +/** + * Sets the document field used as the document reference. Every document must have this field. + * The type of this field in the document should be a string, if it is not a string it will be + * coerced into a string by calling toString. + * + * The default ref is 'id'. + * + * The ref should _not_ be changed during indexing, it should be set before any documents are + * added to the index. Changing it during indexing can lead to inconsistent results. + * + * @param {string} ref - The name of the reference field in the document. + */ +lunr.Builder.prototype.ref = function (ref) { + this._ref = ref +} + +/** + * Adds a field to the list of document fields that will be indexed. Every document being + * indexed should have this field. Null values for this field in indexed documents will + * not cause errors but will limit the chance of that document being retrieved by searches. + * + * All fields should be added before adding documents to the index. Adding fields after + * a document has been indexed will have no effect on already indexed documents. + * + * @param {string} field - The name of a field to index in all documents. + */ +lunr.Builder.prototype.field = function (field) { + this._fields.push(field) +} + +/** + * A parameter to tune the amount of field length normalisation that is applied when + * calculating relevance scores. A value of 0 will completely disable any normalisation + * and a value of 1 will fully normalise field lengths. The default is 0.75. Values of b + * will be clamped to the range 0 - 1. + * + * @param {number} number - The value to set for this tuning parameter. + */ +lunr.Builder.prototype.b = function (number) { + if (number < 0) { + this._b = 0 + } else if (number > 1) { + this._b = 1 + } else { + this._b = number + } +} + +/** + * A parameter that controls the speed at which a rise in term frequency results in term + * frequency saturation. The default value is 1.2. Setting this to a higher value will give + * slower saturation levels, a lower value will result in quicker saturation. + * + * @param {number} number - The value to set for this tuning parameter. + */ +lunr.Builder.prototype.k1 = function (number) { + this._k1 = number +} + +/** + * Adds a document to the index. + * + * Before adding fields to the index the index should have been fully setup, with the document + * ref and all fields to index already having been specified. + * + * The document must have a field name as specified by the ref (by default this is 'id') and + * it should have all fields defined for indexing, though null or undefined values will not + * cause errors. + * + * @param {object} doc - The document to add to the index. + */ +lunr.Builder.prototype.add = function (doc) { + var docRef = doc[this._ref] + + this.documentCount += 1 + + for (var i = 0; i < this._fields.length; i++) { + var fieldName = this._fields[i], + field = doc[fieldName], + tokens = this.tokenizer(field), + terms = this.pipeline.run(tokens), + fieldRef = new lunr.FieldRef (docRef, fieldName), + fieldTerms = Object.create(null) + + this.fieldTermFrequencies[fieldRef] = fieldTerms + this.fieldLengths[fieldRef] = 0 + + // store the length of this field for this document + this.fieldLengths[fieldRef] += terms.length + + // calculate term frequencies for this field + for (var j = 0; j < terms.length; j++) { + var term = terms[j] + + if (fieldTerms[term] == undefined) { + fieldTerms[term] = 0 + } + + fieldTerms[term] += 1 + + // add to inverted index + // create an initial posting if one doesn't exist + if (this.invertedIndex[term] == undefined) { + var posting = Object.create(null) + posting["_index"] = this.termIndex + this.termIndex += 1 + + for (var k = 0; k < this._fields.length; k++) { + posting[this._fields[k]] = Object.create(null) + } + + this.invertedIndex[term] = posting + } + + // add an entry for this term/fieldName/docRef to the invertedIndex + if (this.invertedIndex[term][fieldName][docRef] == undefined) { + this.invertedIndex[term][fieldName][docRef] = Object.create(null) + } + + // store all whitelisted metadata about this token in the + // inverted index + for (var l = 0; l < this.metadataWhitelist.length; l++) { + var metadataKey = this.metadataWhitelist[l], + metadata = term.metadata[metadataKey] + + if (this.invertedIndex[term][fieldName][docRef][metadataKey] == undefined) { + this.invertedIndex[term][fieldName][docRef][metadataKey] = [] + } + + this.invertedIndex[term][fieldName][docRef][metadataKey].push(metadata) + } + } + + } +} + +/** + * Calculates the average document length for this index + * + * @private + */ +lunr.Builder.prototype.calculateAverageFieldLengths = function () { + + var fieldRefs = Object.keys(this.fieldLengths), + numberOfFields = fieldRefs.length, + accumulator = {}, + documentsWithField = {} + + for (var i = 0; i < numberOfFields; i++) { + var fieldRef = lunr.FieldRef.fromString(fieldRefs[i]), + field = fieldRef.fieldName + + documentsWithField[field] || (documentsWithField[field] = 0) + documentsWithField[field] += 1 + + accumulator[field] || (accumulator[field] = 0) + accumulator[field] += this.fieldLengths[fieldRef] + } + + for (var i = 0; i < this._fields.length; i++) { + var field = this._fields[i] + accumulator[field] = accumulator[field] / documentsWithField[field] + } + + this.averageFieldLength = accumulator +} + +/** + * Builds a vector space model of every document using lunr.Vector + * + * @private + */ +lunr.Builder.prototype.createFieldVectors = function () { + var fieldVectors = {}, + fieldRefs = Object.keys(this.fieldTermFrequencies), + fieldRefsLength = fieldRefs.length + + for (var i = 0; i < fieldRefsLength; i++) { + var fieldRef = lunr.FieldRef.fromString(fieldRefs[i]), + field = fieldRef.fieldName, + fieldLength = this.fieldLengths[fieldRef], + fieldVector = new lunr.Vector, + termFrequencies = this.fieldTermFrequencies[fieldRef], + terms = Object.keys(termFrequencies), + termsLength = terms.length + + for (var j = 0; j < termsLength; j++) { + var term = terms[j], + tf = termFrequencies[term], + termIndex = this.invertedIndex[term]._index, + idf = lunr.idf(this.invertedIndex[term], this.documentCount), + score = idf * ((this._k1 + 1) * tf) / (this._k1 * (1 - this._b + this._b * (fieldLength / this.averageFieldLength[field])) + tf), + scoreWithPrecision = Math.round(score * 1000) / 1000 + // Converts 1.23456789 to 1.234. + // Reducing the precision so that the vectors take up less + // space when serialized. Doing it now so that they behave + // the same before and after serialization. Also, this is + // the fastest approach to reducing a number's precision in + // JavaScript. + + fieldVector.insert(termIndex, scoreWithPrecision) + } + + fieldVectors[fieldRef] = fieldVector + } + + this.fieldVectors = fieldVectors +} + +/** + * Creates a token set of all tokens in the index using lunr.TokenSet + * + * @private + */ +lunr.Builder.prototype.createTokenSet = function () { + this.tokenSet = lunr.TokenSet.fromArray( + Object.keys(this.invertedIndex).sort() + ) +} + +/** + * Builds the index, creating an instance of lunr.Index. + * + * This completes the indexing process and should only be called + * once all documents have been added to the index. + * + * @private + * @returns {lunr.Index} + */ +lunr.Builder.prototype.build = function () { + this.calculateAverageFieldLengths() + this.createFieldVectors() + this.createTokenSet() + + return new lunr.Index({ + invertedIndex: this.invertedIndex, + fieldVectors: this.fieldVectors, + tokenSet: this.tokenSet, + fields: this._fields, + pipeline: this.searchPipeline + }) +} + +/** + * Applies a plugin to the index builder. + * + * A plugin is a function that is called with the index builder as its context. + * Plugins can be used to customize or extend the behavior of the index + * in some way. A plugin is just a function, that encapsulated the custom + * behavior that should be applied when building the index. + * + * The plugin function will be called with the index builder as its argument, additional + * arguments can also be passed when calling use. The function will be called + * with the index builder as its context. + * + * @param {Function} plugin The plugin to apply. + */ +lunr.Builder.prototype.use = function (fn) { + var args = Array.prototype.slice.call(arguments, 1) + args.unshift(this) + fn.apply(this, args) +} +/** + * Contains and collects metadata about a matching document. + * A single instance of lunr.MatchData is returned as part of every + * lunr.Index~Result. + * + * @constructor + * @param {string} term - The term this match data is associated with + * @param {string} field - The field in which the term was found + * @param {object} metadata - The metadata recorded about this term in this field + * @property {object} metadata - A cloned collection of metadata associated with this document. + * @see {@link lunr.Index~Result} + */ +lunr.MatchData = function (term, field, metadata) { + var clonedMetadata = Object.create(null), + metadataKeys = Object.keys(metadata) + + // Cloning the metadata to prevent the original + // being mutated during match data combination. + // Metadata is kept in an array within the inverted + // index so cloning the data can be done with + // Array#slice + for (var i = 0; i < metadataKeys.length; i++) { + var key = metadataKeys[i] + clonedMetadata[key] = metadata[key].slice() + } + + this.metadata = Object.create(null) + this.metadata[term] = Object.create(null) + this.metadata[term][field] = clonedMetadata +} + +/** + * An instance of lunr.MatchData will be created for every term that matches a + * document. However only one instance is required in a lunr.Index~Result. This + * method combines metadata from another instance of lunr.MatchData with this + * objects metadata. + * + * @param {lunr.MatchData} otherMatchData - Another instance of match data to merge with this one. + * @see {@link lunr.Index~Result} + */ +lunr.MatchData.prototype.combine = function (otherMatchData) { + var terms = Object.keys(otherMatchData.metadata) + + for (var i = 0; i < terms.length; i++) { + var term = terms[i], + fields = Object.keys(otherMatchData.metadata[term]) + + if (this.metadata[term] == undefined) { + this.metadata[term] = Object.create(null) + } + + for (var j = 0; j < fields.length; j++) { + var field = fields[j], + keys = Object.keys(otherMatchData.metadata[term][field]) + + if (this.metadata[term][field] == undefined) { + this.metadata[term][field] = Object.create(null) + } + + for (var k = 0; k < keys.length; k++) { + var key = keys[k] + + if (this.metadata[term][field][key] == undefined) { + this.metadata[term][field][key] = otherMatchData.metadata[term][field][key] + } else { + this.metadata[term][field][key] = this.metadata[term][field][key].concat(otherMatchData.metadata[term][field][key]) + } + + } + } + } +} +/** + * A lunr.Query provides a programmatic way of defining queries to be performed + * against a {@link lunr.Index}. + * + * Prefer constructing a lunr.Query using the {@link lunr.Index#query} method + * so the query object is pre-initialized with the right index fields. + * + * @constructor + * @property {lunr.Query~Clause[]} clauses - An array of query clauses. + * @property {string[]} allFields - An array of all available fields in a lunr.Index. + */ +lunr.Query = function (allFields) { + this.clauses = [] + this.allFields = allFields +} + +/** + * Constants for indicating what kind of automatic wildcard insertion will be used when constructing a query clause. + * + * This allows wildcards to be added to the beginning and end of a term without having to manually do any string + * concatenation. + * + * The wildcard constants can be bitwise combined to select both leading and trailing wildcards. + * + * @constant + * @default + * @property {number} wildcard.NONE - The term will have no wildcards inserted, this is the default behavior + * @property {number} wildcard.LEADING - Prepend the term with a wildcard, unless a leading wildcard already exists + * @property {number} wildcard.TRAILING - Append a wildcard to the term, unless a trailing wildcard already exists + * @see lunr.Query~Clause + * @see lunr.Query#clause + * @see lunr.Query#term + * @example query term with trailing wildcard + * query.term('foo', { wildcard: lunr.Query.wildcard.TRAILING }) + * @example query term with leading and trailing wildcard + * query.term('foo', { + * wildcard: lunr.Query.wildcard.LEADING | lunr.Query.wildcard.TRAILING + * }) + */ +lunr.Query.wildcard = new String ("*") +lunr.Query.wildcard.NONE = 0 +lunr.Query.wildcard.LEADING = 1 +lunr.Query.wildcard.TRAILING = 2 + +/** + * A single clause in a {@link lunr.Query} contains a term and details on how to + * match that term against a {@link lunr.Index}. + * + * @typedef {Object} lunr.Query~Clause + * @property {string[]} fields - The fields in an index this clause should be matched against. + * @property {number} [boost=1] - Any boost that should be applied when matching this clause. + * @property {number} [editDistance] - Whether the term should have fuzzy matching applied, and how fuzzy the match should be. + * @property {boolean} [usePipeline] - Whether the term should be passed through the search pipeline. + * @property {number} [wildcard=0] - Whether the term should have wildcards appended or prepended. + */ + +/** + * Adds a {@link lunr.Query~Clause} to this query. + * + * Unless the clause contains the fields to be matched all fields will be matched. In addition + * a default boost of 1 is applied to the clause. + * + * @param {lunr.Query~Clause} clause - The clause to add to this query. + * @see lunr.Query~Clause + * @returns {lunr.Query} + */ +lunr.Query.prototype.clause = function (clause) { + if (!('fields' in clause)) { + clause.fields = this.allFields + } + + if (!('boost' in clause)) { + clause.boost = 1 + } + + if (!('usePipeline' in clause)) { + clause.usePipeline = true + } + + if (!('wildcard' in clause)) { + clause.wildcard = lunr.Query.wildcard.NONE + } + + if ((clause.wildcard & lunr.Query.wildcard.LEADING) && (clause.term.charAt(0) != lunr.Query.wildcard)) { + clause.term = "*" + clause.term + } + + if ((clause.wildcard & lunr.Query.wildcard.TRAILING) && (clause.term.slice(-1) != lunr.Query.wildcard)) { + clause.term = "" + clause.term + "*" + } + + this.clauses.push(clause) + + return this +} + +/** + * Adds a term to the current query, under the covers this will create a {@link lunr.Query~Clause} + * to the list of clauses that make up this query. + * + * @param {string} term - The term to add to the query. + * @param {Object} [options] - Any additional properties to add to the query clause. + * @returns {lunr.Query} + * @see lunr.Query#clause + * @see lunr.Query~Clause + * @example adding a single term to a query + * query.term("foo") + * @example adding a single term to a query and specifying search fields, term boost and automatic trailing wildcard + * query.term("foo", { + * fields: ["title"], + * boost: 10, + * wildcard: lunr.Query.wildcard.TRAILING + * }) + */ +lunr.Query.prototype.term = function (term, options) { + var clause = options || {} + clause.term = term + + this.clause(clause) + + return this +} +lunr.QueryParseError = function (message, start, end) { + this.name = "QueryParseError" + this.message = message + this.start = start + this.end = end +} + +lunr.QueryParseError.prototype = new Error +lunr.QueryLexer = function (str) { + this.lexemes = [] + this.str = str + this.length = str.length + this.pos = 0 + this.start = 0 + this.escapeCharPositions = [] +} + +lunr.QueryLexer.prototype.run = function () { + var state = lunr.QueryLexer.lexText + + while (state) { + state = state(this) + } +} + +lunr.QueryLexer.prototype.sliceString = function () { + var subSlices = [], + sliceStart = this.start, + sliceEnd = this.pos + + for (var i = 0; i < this.escapeCharPositions.length; i++) { + sliceEnd = this.escapeCharPositions[i] + subSlices.push(this.str.slice(sliceStart, sliceEnd)) + sliceStart = sliceEnd + 1 + } + + subSlices.push(this.str.slice(sliceStart, this.pos)) + this.escapeCharPositions.length = 0 + + return subSlices.join('') +} + +lunr.QueryLexer.prototype.emit = function (type) { + this.lexemes.push({ + type: type, + str: this.sliceString(), + start: this.start, + end: this.pos + }) + + this.start = this.pos +} + +lunr.QueryLexer.prototype.escapeCharacter = function () { + this.escapeCharPositions.push(this.pos - 1) + this.pos += 1 +} + +lunr.QueryLexer.prototype.next = function () { + if (this.pos >= this.length) { + return lunr.QueryLexer.EOS + } + + var char = this.str.charAt(this.pos) + this.pos += 1 + return char +} + +lunr.QueryLexer.prototype.width = function () { + return this.pos - this.start +} + +lunr.QueryLexer.prototype.ignore = function () { + if (this.start == this.pos) { + this.pos += 1 + } + + this.start = this.pos +} + +lunr.QueryLexer.prototype.backup = function () { + this.pos -= 1 +} + +lunr.QueryLexer.prototype.acceptDigitRun = function () { + var char, charCode + + do { + char = this.next() + charCode = char.charCodeAt(0) + } while (charCode > 47 && charCode < 58) + + if (char != lunr.QueryLexer.EOS) { + this.backup() + } +} + +lunr.QueryLexer.prototype.more = function () { + return this.pos < this.length +} + +lunr.QueryLexer.EOS = 'EOS' +lunr.QueryLexer.FIELD = 'FIELD' +lunr.QueryLexer.TERM = 'TERM' +lunr.QueryLexer.EDIT_DISTANCE = 'EDIT_DISTANCE' +lunr.QueryLexer.BOOST = 'BOOST' + +lunr.QueryLexer.lexField = function (lexer) { + lexer.backup() + lexer.emit(lunr.QueryLexer.FIELD) + lexer.ignore() + return lunr.QueryLexer.lexText +} + +lunr.QueryLexer.lexTerm = function (lexer) { + if (lexer.width() > 1) { + lexer.backup() + lexer.emit(lunr.QueryLexer.TERM) + } + + lexer.ignore() + + if (lexer.more()) { + return lunr.QueryLexer.lexText + } +} + +lunr.QueryLexer.lexEditDistance = function (lexer) { + lexer.ignore() + lexer.acceptDigitRun() + lexer.emit(lunr.QueryLexer.EDIT_DISTANCE) + return lunr.QueryLexer.lexText +} + +lunr.QueryLexer.lexBoost = function (lexer) { + lexer.ignore() + lexer.acceptDigitRun() + lexer.emit(lunr.QueryLexer.BOOST) + return lunr.QueryLexer.lexText +} + +lunr.QueryLexer.lexEOS = function (lexer) { + if (lexer.width() > 0) { + lexer.emit(lunr.QueryLexer.TERM) + } +} + +// This matches the separator used when tokenising fields +// within a document. These should match otherwise it is +// not possible to search for some tokens within a document. +// +// It is possible for the user to change the separator on the +// tokenizer so it _might_ clash with any other of the special +// characters already used within the search string, e.g. :. +// +// This means that it is possible to change the separator in +// such a way that makes some words unsearchable using a search +// string. +lunr.QueryLexer.termSeparator = lunr.tokenizer.separator + +lunr.QueryLexer.lexText = function (lexer) { + while (true) { + var char = lexer.next() + + if (char == lunr.QueryLexer.EOS) { + return lunr.QueryLexer.lexEOS + } + + // Escape character is '\' + if (char.charCodeAt(0) == 92) { + lexer.escapeCharacter() + continue + } + + if (char == ":") { + return lunr.QueryLexer.lexField + } + + if (char == "~") { + lexer.backup() + if (lexer.width() > 0) { + lexer.emit(lunr.QueryLexer.TERM) + } + return lunr.QueryLexer.lexEditDistance + } + + if (char == "^") { + lexer.backup() + if (lexer.width() > 0) { + lexer.emit(lunr.QueryLexer.TERM) + } + return lunr.QueryLexer.lexBoost + } + + if (char.match(lunr.QueryLexer.termSeparator)) { + return lunr.QueryLexer.lexTerm + } + } +} + +lunr.QueryParser = function (str, query) { + this.lexer = new lunr.QueryLexer (str) + this.query = query + this.currentClause = {} + this.lexemeIdx = 0 +} + +lunr.QueryParser.prototype.parse = function () { + this.lexer.run() + this.lexemes = this.lexer.lexemes + + var state = lunr.QueryParser.parseFieldOrTerm + + while (state) { + state = state(this) + } + + return this.query +} + +lunr.QueryParser.prototype.peekLexeme = function () { + return this.lexemes[this.lexemeIdx] +} + +lunr.QueryParser.prototype.consumeLexeme = function () { + var lexeme = this.peekLexeme() + this.lexemeIdx += 1 + return lexeme +} + +lunr.QueryParser.prototype.nextClause = function () { + var completedClause = this.currentClause + this.query.clause(completedClause) + this.currentClause = {} +} + +lunr.QueryParser.parseFieldOrTerm = function (parser) { + var lexeme = parser.peekLexeme() + + if (lexeme == undefined) { + return + } + + switch (lexeme.type) { + case lunr.QueryLexer.FIELD: + return lunr.QueryParser.parseField + case lunr.QueryLexer.TERM: + return lunr.QueryParser.parseTerm + default: + var errorMessage = "expected either a field or a term, found " + lexeme.type + + if (lexeme.str.length >= 1) { + errorMessage += " with value '" + lexeme.str + "'" + } + + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } +} + +lunr.QueryParser.parseField = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + if (parser.query.allFields.indexOf(lexeme.str) == -1) { + var possibleFields = parser.query.allFields.map(function (f) { return "'" + f + "'" }).join(', '), + errorMessage = "unrecognized field '" + lexeme.str + "', possible fields: " + possibleFields + + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + parser.currentClause.fields = [lexeme.str] + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + var errorMessage = "expecting term, found nothing" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + return lunr.QueryParser.parseTerm + default: + var errorMessage = "expecting term, found '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseTerm = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + parser.currentClause.term = lexeme.str.toLowerCase() + + if (lexeme.str.indexOf("*") != -1) { + parser.currentClause.usePipeline = false + } + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + parser.nextClause() + return + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + parser.nextClause() + return lunr.QueryParser.parseTerm + case lunr.QueryLexer.FIELD: + parser.nextClause() + return lunr.QueryParser.parseField + case lunr.QueryLexer.EDIT_DISTANCE: + return lunr.QueryParser.parseEditDistance + case lunr.QueryLexer.BOOST: + return lunr.QueryParser.parseBoost + default: + var errorMessage = "Unexpected lexeme type '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseEditDistance = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + var editDistance = parseInt(lexeme.str, 10) + + if (isNaN(editDistance)) { + var errorMessage = "edit distance must be numeric" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + parser.currentClause.editDistance = editDistance + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + parser.nextClause() + return + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + parser.nextClause() + return lunr.QueryParser.parseTerm + case lunr.QueryLexer.FIELD: + parser.nextClause() + return lunr.QueryParser.parseField + case lunr.QueryLexer.EDIT_DISTANCE: + return lunr.QueryParser.parseEditDistance + case lunr.QueryLexer.BOOST: + return lunr.QueryParser.parseBoost + default: + var errorMessage = "Unexpected lexeme type '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseBoost = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + var boost = parseInt(lexeme.str, 10) + + if (isNaN(boost)) { + var errorMessage = "boost must be numeric" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + parser.currentClause.boost = boost + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + parser.nextClause() + return + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + parser.nextClause() + return lunr.QueryParser.parseTerm + case lunr.QueryLexer.FIELD: + parser.nextClause() + return lunr.QueryParser.parseField + case lunr.QueryLexer.EDIT_DISTANCE: + return lunr.QueryParser.parseEditDistance + case lunr.QueryLexer.BOOST: + return lunr.QueryParser.parseBoost + default: + var errorMessage = "Unexpected lexeme type '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + + /** + * export the module via AMD, CommonJS or as a browser global + * Export code from https://github.com/umdjs/umd/blob/master/returnExports.js + */ + ;(function (root, factory) { + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define(factory) + } else if (typeof exports === 'object') { + /** + * Node. Does not work with strict CommonJS, but + * only CommonJS-like environments that support module.exports, + * like Node. + */ + module.exports = factory() + } else { + // Browser globals (root is window) + root.lunr = factory() + } + }(this, function () { + /** + * Just return a value to define the module export. + * This example returns an object, but the module + * can return a function as the exported value. + */ + return lunr + })) +})(); diff --git a/csharp/ApiDocs/_exported_templates/default/styles/lunr.min.js b/csharp/ApiDocs/_exported_templates/default/styles/lunr.min.js new file mode 100644 index 0000000000..9ba9309986 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/lunr.min.js @@ -0,0 +1 @@ +(function(){var lunr=function(config){var builder=new lunr.Builder;builder.pipeline.add(lunr.trimmer,lunr.stopWordFilter,lunr.stemmer);builder.searchPipeline.add(lunr.stemmer);config.call(builder,builder);return builder.build()};lunr.version="2.1.2";lunr.utils={};lunr.utils.warn=function(global){return function(message){if(global.console&&console.warn){console.warn(message)}}}(this);lunr.utils.asString=function(obj){if(obj===void 0||obj===null){return""}else{return obj.toString()}};lunr.FieldRef=function(docRef,fieldName){this.docRef=docRef;this.fieldName=fieldName;this._stringValue=fieldName+lunr.FieldRef.joiner+docRef};lunr.FieldRef.joiner="/";lunr.FieldRef.fromString=function(s){var n=s.indexOf(lunr.FieldRef.joiner);if(n===-1){throw"malformed field ref string"}var fieldRef=s.slice(0,n),docRef=s.slice(n+1);return new lunr.FieldRef(docRef,fieldRef)};lunr.FieldRef.prototype.toString=function(){return this._stringValue};lunr.idf=function(posting,documentCount){var documentsWithTerm=0;for(var fieldName in posting){if(fieldName=="_index")continue;documentsWithTerm+=Object.keys(posting[fieldName]).length}var x=(documentCount-documentsWithTerm+.5)/(documentsWithTerm+.5);return Math.log(1+Math.abs(x))};lunr.Token=function(str,metadata){this.str=str||"";this.metadata=metadata||{}};lunr.Token.prototype.toString=function(){return this.str};lunr.Token.prototype.update=function(fn){this.str=fn(this.str,this.metadata);return this};lunr.Token.prototype.clone=function(fn){fn=fn||function(s){return s};return new lunr.Token(fn(this.str,this.metadata),this.metadata)};lunr.tokenizer=function(obj){if(obj==null||obj==undefined){return[]}if(Array.isArray(obj)){return obj.map((function(t){return new lunr.Token(lunr.utils.asString(t).toLowerCase())}))}var str=obj.toString().trim().toLowerCase(),len=str.length,tokens=[];for(var sliceEnd=0,sliceStart=0;sliceEnd<=len;sliceEnd++){var char=str.charAt(sliceEnd),sliceLength=sliceEnd-sliceStart;if(char.match(lunr.tokenizer.separator)||sliceEnd==len){if(sliceLength>0){tokens.push(new lunr.Token(str.slice(sliceStart,sliceEnd),{position:[sliceStart,sliceLength],index:tokens.length}))}sliceStart=sliceEnd+1}}return tokens};lunr.tokenizer.separator=/[\s\-]+/;lunr.Pipeline=function(){this._stack=[]};lunr.Pipeline.registeredFunctions=Object.create(null);lunr.Pipeline.registerFunction=function(fn,label){if(label in this.registeredFunctions){lunr.utils.warn("Overwriting existing registered function: "+label)}fn.label=label;lunr.Pipeline.registeredFunctions[fn.label]=fn};lunr.Pipeline.warnIfFunctionNotRegistered=function(fn){var isRegistered=fn.label&&fn.label in this.registeredFunctions;if(!isRegistered){lunr.utils.warn("Function is not registered with pipeline. This may cause problems when serializing the index.\n",fn)}};lunr.Pipeline.load=function(serialized){var pipeline=new lunr.Pipeline;serialized.forEach((function(fnName){var fn=lunr.Pipeline.registeredFunctions[fnName];if(fn){pipeline.add(fn)}else{throw new Error("Cannot load unregistered function: "+fnName)}}));return pipeline};lunr.Pipeline.prototype.add=function(){var fns=Array.prototype.slice.call(arguments);fns.forEach((function(fn){lunr.Pipeline.warnIfFunctionNotRegistered(fn);this._stack.push(fn)}),this)};lunr.Pipeline.prototype.after=function(existingFn,newFn){lunr.Pipeline.warnIfFunctionNotRegistered(newFn);var pos=this._stack.indexOf(existingFn);if(pos==-1){throw new Error("Cannot find existingFn")}pos=pos+1;this._stack.splice(pos,0,newFn)};lunr.Pipeline.prototype.before=function(existingFn,newFn){lunr.Pipeline.warnIfFunctionNotRegistered(newFn);var pos=this._stack.indexOf(existingFn);if(pos==-1){throw new Error("Cannot find existingFn")}this._stack.splice(pos,0,newFn)};lunr.Pipeline.prototype.remove=function(fn){var pos=this._stack.indexOf(fn);if(pos==-1){return}this._stack.splice(pos,1)};lunr.Pipeline.prototype.run=function(tokens){var stackLength=this._stack.length;for(var i=0;i1){if(pivotIndexindex){end=pivotPoint}if(pivotIndex==index){break}sliceLength=end-start;pivotPoint=start+Math.floor(sliceLength/2);pivotIndex=this.elements[pivotPoint*2]}if(pivotIndex==index){return pivotPoint*2}if(pivotIndex>index){return pivotPoint*2}if(pivotIndexbVal){j+=2}else if(aVal==bVal){dotProduct+=a[i+1]*b[j+1];i+=2;j+=2}}return dotProduct};lunr.Vector.prototype.similarity=function(otherVector){return this.dot(otherVector)/(this.magnitude()*otherVector.magnitude())};lunr.Vector.prototype.toArray=function(){var output=new Array(this.elements.length/2);for(var i=1,j=0;i0){var char=frame.str.charAt(0),noEditNode;if(char in frame.node.edges){noEditNode=frame.node.edges[char]}else{noEditNode=new lunr.TokenSet;frame.node.edges[char]=noEditNode}if(frame.str.length==1){noEditNode.final=true}else{stack.push({node:noEditNode,editsRemaining:frame.editsRemaining,str:frame.str.slice(1)})}}if(frame.editsRemaining>0&&frame.str.length>1){var char=frame.str.charAt(1),deletionNode;if(char in frame.node.edges){deletionNode=frame.node.edges[char]}else{deletionNode=new lunr.TokenSet;frame.node.edges[char]=deletionNode}if(frame.str.length<=2){deletionNode.final=true}else{stack.push({node:deletionNode,editsRemaining:frame.editsRemaining-1,str:frame.str.slice(2)})}}if(frame.editsRemaining>0&&frame.str.length==1){frame.node.final=true}if(frame.editsRemaining>0&&frame.str.length>=1){if("*"in frame.node.edges){var substitutionNode=frame.node.edges["*"]}else{var substitutionNode=new lunr.TokenSet;frame.node.edges["*"]=substitutionNode}if(frame.str.length==1){substitutionNode.final=true}else{stack.push({node:substitutionNode,editsRemaining:frame.editsRemaining-1,str:frame.str.slice(1)})}}if(frame.editsRemaining>0){if("*"in frame.node.edges){var insertionNode=frame.node.edges["*"]}else{var insertionNode=new lunr.TokenSet;frame.node.edges["*"]=insertionNode}if(frame.str.length==0){insertionNode.final=true}else{stack.push({node:insertionNode,editsRemaining:frame.editsRemaining-1,str:frame.str})}}if(frame.editsRemaining>0&&frame.str.length>1){var charA=frame.str.charAt(0),charB=frame.str.charAt(1),transposeNode;if(charB in frame.node.edges){transposeNode=frame.node.edges[charB]}else{transposeNode=new lunr.TokenSet;frame.node.edges[charB]=transposeNode}if(frame.str.length==1){transposeNode.final=true}else{stack.push({node:transposeNode,editsRemaining:frame.editsRemaining-1,str:charA+frame.str.slice(2)})}}}return root};lunr.TokenSet.fromString=function(str){var node=new lunr.TokenSet,root=node,wildcardFound=false;for(var i=0,len=str.length;i=downTo;i--){var node=this.uncheckedNodes[i],childKey=node.child.toString();if(childKey in this.minimizedNodes){node.parent.edges[node.char]=this.minimizedNodes[childKey]}else{node.child._str=childKey;this.minimizedNodes[childKey]=node.child}this.uncheckedNodes.pop()}};lunr.Index=function(attrs){this.invertedIndex=attrs.invertedIndex;this.fieldVectors=attrs.fieldVectors;this.tokenSet=attrs.tokenSet;this.fields=attrs.fields;this.pipeline=attrs.pipeline};lunr.Index.prototype.search=function(queryString){return this.query((function(query){var parser=new lunr.QueryParser(queryString,query);parser.parse()}))};lunr.Index.prototype.query=function(fn){var query=new lunr.Query(this.fields),matchingFields=Object.create(null),queryVectors=Object.create(null);fn.call(query,query);for(var i=0;i1){this._b=1}else{this._b=number}};lunr.Builder.prototype.k1=function(number){this._k1=number};lunr.Builder.prototype.add=function(doc){var docRef=doc[this._ref];this.documentCount+=1;for(var i=0;i=this.length){return lunr.QueryLexer.EOS}var char=this.str.charAt(this.pos);this.pos+=1;return char};lunr.QueryLexer.prototype.width=function(){return this.pos-this.start};lunr.QueryLexer.prototype.ignore=function(){if(this.start==this.pos){this.pos+=1}this.start=this.pos};lunr.QueryLexer.prototype.backup=function(){this.pos-=1};lunr.QueryLexer.prototype.acceptDigitRun=function(){var char,charCode;do{char=this.next();charCode=char.charCodeAt(0)}while(charCode>47&&charCode<58);if(char!=lunr.QueryLexer.EOS){this.backup()}};lunr.QueryLexer.prototype.more=function(){return this.pos1){lexer.backup();lexer.emit(lunr.QueryLexer.TERM)}lexer.ignore();if(lexer.more()){return lunr.QueryLexer.lexText}};lunr.QueryLexer.lexEditDistance=function(lexer){lexer.ignore();lexer.acceptDigitRun();lexer.emit(lunr.QueryLexer.EDIT_DISTANCE);return lunr.QueryLexer.lexText};lunr.QueryLexer.lexBoost=function(lexer){lexer.ignore();lexer.acceptDigitRun();lexer.emit(lunr.QueryLexer.BOOST);return lunr.QueryLexer.lexText};lunr.QueryLexer.lexEOS=function(lexer){if(lexer.width()>0){lexer.emit(lunr.QueryLexer.TERM)}};lunr.QueryLexer.termSeparator=lunr.tokenizer.separator;lunr.QueryLexer.lexText=function(lexer){while(true){var char=lexer.next();if(char==lunr.QueryLexer.EOS){return lunr.QueryLexer.lexEOS}if(char.charCodeAt(0)==92){lexer.escapeCharacter();continue}if(char==":"){return lunr.QueryLexer.lexField}if(char=="~"){lexer.backup();if(lexer.width()>0){lexer.emit(lunr.QueryLexer.TERM)}return lunr.QueryLexer.lexEditDistance}if(char=="^"){lexer.backup();if(lexer.width()>0){lexer.emit(lunr.QueryLexer.TERM)}return lunr.QueryLexer.lexBoost}if(char.match(lunr.QueryLexer.termSeparator)){return lunr.QueryLexer.lexTerm}}};lunr.QueryParser=function(str,query){this.lexer=new lunr.QueryLexer(str);this.query=query;this.currentClause={};this.lexemeIdx=0};lunr.QueryParser.prototype.parse=function(){this.lexer.run();this.lexemes=this.lexer.lexemes;var state=lunr.QueryParser.parseFieldOrTerm;while(state){state=state(this)}return this.query};lunr.QueryParser.prototype.peekLexeme=function(){return this.lexemes[this.lexemeIdx]};lunr.QueryParser.prototype.consumeLexeme=function(){var lexeme=this.peekLexeme();this.lexemeIdx+=1;return lexeme};lunr.QueryParser.prototype.nextClause=function(){var completedClause=this.currentClause;this.query.clause(completedClause);this.currentClause={}};lunr.QueryParser.parseFieldOrTerm=function(parser){var lexeme=parser.peekLexeme();if(lexeme==undefined){return}switch(lexeme.type){case lunr.QueryLexer.FIELD:return lunr.QueryParser.parseField;case lunr.QueryLexer.TERM:return lunr.QueryParser.parseTerm;default:var errorMessage="expected either a field or a term, found "+lexeme.type;if(lexeme.str.length>=1){errorMessage+=" with value '"+lexeme.str+"'"}throw new lunr.QueryParseError(errorMessage,lexeme.start,lexeme.end)}};lunr.QueryParser.parseField=function(parser){var lexeme=parser.consumeLexeme();if(lexeme==undefined){return}if(parser.query.allFields.indexOf(lexeme.str)==-1){var possibleFields=parser.query.allFields.map((function(f){return"'"+f+"'"})).join(", "),errorMessage="unrecognized field '"+lexeme.str+"', possible fields: "+possibleFields;throw new lunr.QueryParseError(errorMessage,lexeme.start,lexeme.end)}parser.currentClause.fields=[lexeme.str];var nextLexeme=parser.peekLexeme();if(nextLexeme==undefined){var errorMessage="expecting term, found nothing";throw new lunr.QueryParseError(errorMessage,lexeme.start,lexeme.end)}switch(nextLexeme.type){case lunr.QueryLexer.TERM:return lunr.QueryParser.parseTerm;default:var errorMessage="expecting term, found '"+nextLexeme.type+"'";throw new lunr.QueryParseError(errorMessage,nextLexeme.start,nextLexeme.end)}};lunr.QueryParser.parseTerm=function(parser){var lexeme=parser.consumeLexeme();if(lexeme==undefined){return}parser.currentClause.term=lexeme.str.toLowerCase();if(lexeme.str.indexOf("*")!=-1){parser.currentClause.usePipeline=false}var nextLexeme=parser.peekLexeme();if(nextLexeme==undefined){parser.nextClause();return}switch(nextLexeme.type){case lunr.QueryLexer.TERM:parser.nextClause();return lunr.QueryParser.parseTerm;case lunr.QueryLexer.FIELD:parser.nextClause();return lunr.QueryParser.parseField;case lunr.QueryLexer.EDIT_DISTANCE:return lunr.QueryParser.parseEditDistance;case lunr.QueryLexer.BOOST:return lunr.QueryParser.parseBoost;default:var errorMessage="Unexpected lexeme type '"+nextLexeme.type+"'";throw new lunr.QueryParseError(errorMessage,nextLexeme.start,nextLexeme.end)}};lunr.QueryParser.parseEditDistance=function(parser){var lexeme=parser.consumeLexeme();if(lexeme==undefined){return}var editDistance=parseInt(lexeme.str,10);if(isNaN(editDistance)){var errorMessage="edit distance must be numeric";throw new lunr.QueryParseError(errorMessage,lexeme.start,lexeme.end)}parser.currentClause.editDistance=editDistance;var nextLexeme=parser.peekLexeme();if(nextLexeme==undefined){parser.nextClause();return}switch(nextLexeme.type){case lunr.QueryLexer.TERM:parser.nextClause();return lunr.QueryParser.parseTerm;case lunr.QueryLexer.FIELD:parser.nextClause();return lunr.QueryParser.parseField;case lunr.QueryLexer.EDIT_DISTANCE:return lunr.QueryParser.parseEditDistance;case lunr.QueryLexer.BOOST:return lunr.QueryParser.parseBoost;default:var errorMessage="Unexpected lexeme type '"+nextLexeme.type+"'";throw new lunr.QueryParseError(errorMessage,nextLexeme.start,nextLexeme.end)}};lunr.QueryParser.parseBoost=function(parser){var lexeme=parser.consumeLexeme();if(lexeme==undefined){return}var boost=parseInt(lexeme.str,10);if(isNaN(boost)){var errorMessage="boost must be numeric";throw new lunr.QueryParseError(errorMessage,lexeme.start,lexeme.end)}parser.currentClause.boost=boost;var nextLexeme=parser.peekLexeme();if(nextLexeme==undefined){parser.nextClause();return}switch(nextLexeme.type){case lunr.QueryLexer.TERM:parser.nextClause();return lunr.QueryParser.parseTerm;case lunr.QueryLexer.FIELD:parser.nextClause();return lunr.QueryParser.parseField;case lunr.QueryLexer.EDIT_DISTANCE:return lunr.QueryParser.parseEditDistance;case lunr.QueryLexer.BOOST:return lunr.QueryParser.parseBoost;default:var errorMessage="Unexpected lexeme type '"+nextLexeme.type+"'";throw new lunr.QueryParseError(errorMessage,nextLexeme.start,nextLexeme.end)}};(function(root,factory){if(typeof define==="function"&&define.amd){define(factory)}else if(typeof exports==="object"){module.exports=factory()}else{root.lunr=factory()}})(this,(function(){return lunr}))})(); diff --git a/orttraining/orttraining/test/python/checkpoint/__init__.py b/csharp/ApiDocs/_exported_templates/default/styles/main.css similarity index 100% rename from orttraining/orttraining/test/python/checkpoint/__init__.py rename to csharp/ApiDocs/_exported_templates/default/styles/main.css diff --git a/csharp/ApiDocs/_exported_templates/default/styles/main.js b/csharp/ApiDocs/_exported_templates/default/styles/main.js new file mode 100644 index 0000000000..aeca70d851 --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/main.js @@ -0,0 +1 @@ +// Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information. diff --git a/csharp/ApiDocs/_exported_templates/default/styles/search-worker.js b/csharp/ApiDocs/_exported_templates/default/styles/search-worker.js new file mode 100644 index 0000000000..60852af4ce --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/styles/search-worker.js @@ -0,0 +1,80 @@ +(function () { + importScripts('lunr.min.js'); + + var lunrIndex; + + var stopWords = null; + var searchData = {}; + + lunr.tokenizer.separator = /[\s\-\.\(\)]+/; + + var stopWordsRequest = new XMLHttpRequest(); + stopWordsRequest.open('GET', '../search-stopwords.json'); + stopWordsRequest.onload = function () { + if (this.status != 200) { + return; + } + stopWords = JSON.parse(this.responseText); + buildIndex(); + } + stopWordsRequest.send(); + + var searchDataRequest = new XMLHttpRequest(); + + searchDataRequest.open('GET', '../index.json'); + searchDataRequest.onload = function () { + if (this.status != 200) { + return; + } + searchData = JSON.parse(this.responseText); + + buildIndex(); + + postMessage({ e: 'index-ready' }); + } + searchDataRequest.send(); + + onmessage = function (oEvent) { + var q = oEvent.data.q; + var hits = lunrIndex.search(q); + var results = []; + hits.forEach(function (hit) { + var item = searchData[hit.ref]; + results.push({ 'href': item.href, 'title': item.title, 'keywords': item.keywords }); + }); + postMessage({ e: 'query-ready', q: q, d: results }); + } + + function buildIndex() { + if (stopWords !== null && !isEmpty(searchData)) { + lunrIndex = lunr(function () { + this.pipeline.remove(lunr.stopWordFilter); + this.ref('href'); + this.field('title', { boost: 50 }); + this.field('keywords', { boost: 20 }); + + for (var prop in searchData) { + if (searchData.hasOwnProperty(prop)) { + this.add(searchData[prop]); + } + } + + var docfxStopWordFilter = lunr.generateStopWordFilter(stopWords); + lunr.Pipeline.registerFunction(docfxStopWordFilter, 'docfxStopWordFilter'); + this.pipeline.add(docfxStopWordFilter); + this.searchPipeline.add(docfxStopWordFilter); + }); + } + } + + function isEmpty(obj) { + if(!obj) return true; + + for (var prop in obj) { + if (obj.hasOwnProperty(prop)) + return false; + } + + return true; + } +})(); diff --git a/csharp/ApiDocs/_exported_templates/default/toc.extension.js b/csharp/ApiDocs/_exported_templates/default/toc.extension.js new file mode 100644 index 0000000000..ba0b6a6feb --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/toc.extension.js @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information. + +/** + * This method will be called at the start of exports.transform in toc.html.js + */ +exports.preTransform = function (model) { + return model; +} + +/** + * This method will be called at the end of exports.transform in toc.html.js + */ +exports.postTransform = function (model) { + return model; +} \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/toc.html.js b/csharp/ApiDocs/_exported_templates/default/toc.html.js new file mode 100644 index 0000000000..8761d781fc --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/toc.html.js @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information. +var extension = require('./toc.extension.js') + +exports.transform = function (model) { + + if (extension && extension.preTransform) { + model = extension.preTransform(model); + } + + transformItem(model, 1); + if (model.items && model.items.length > 0) model.leaf = false; + model.title = "Table of Content"; + model._disableToc = true; + + if (extension && extension.postTransform) { + model = extension.postTransform(model); + } + + return model; + + function transformItem(item, level) { + // set to null incase mustache looks up + item.topicHref = item.topicHref || null; + item.tocHref = item.tocHref || null; + item.name = item.name || null; + + item.level = level; + if (item.items && item.items.length > 0) { + var length = item.items.length; + for (var i = 0; i < length; i++) { + transformItem(item.items[i], level + 1); + }; + } else { + item.items = []; + item.leaf = true; + } + } +} diff --git a/csharp/ApiDocs/_exported_templates/default/toc.html.tmpl b/csharp/ApiDocs/_exported_templates/default/toc.html.tmpl new file mode 100644 index 0000000000..8c02dabcba --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/toc.html.tmpl @@ -0,0 +1,22 @@ +{{!Copyright (c) Microsoft. All rights reserved. Licensed under the MIT license. See LICENSE file in the project root for full license information.}} + +
      +
      + {{^_disableSideFilter}} +
      +
      + + + +
      +
      + {{/_disableSideFilter}} +
      +
      + {{^leaf}} + {{>partials/li}} + {{/leaf}} +
      +
      +
      +
      \ No newline at end of file diff --git a/csharp/ApiDocs/_exported_templates/default/token.json b/csharp/ApiDocs/_exported_templates/default/token.json new file mode 100644 index 0000000000..0c3ca58c7e --- /dev/null +++ b/csharp/ApiDocs/_exported_templates/default/token.json @@ -0,0 +1,62 @@ +{ + "namespacesInSubtitle": "Namespaces", + "classesInSubtitle": "Classes", + "structsInSubtitle": "Structs", + "interfacesInSubtitle": "Interfaces", + "enumsInSubtitle": "Enums", + "delegatesInSubtitle": "Delegates", + "constructorsInSubtitle": "Constructors", + "fieldsInSubtitle": "Fields", + "propertiesInSubtitle": "Properties", + "methodsInSubtitle": "Methods", + "eventsInSubtitle": "Events", + "operatorsInSubtitle": "Operators", + "eiisInSubtitle": "Explicit Interface Implementations", + "functionsInSubtitle": "Functions", + "variablesInSubtitle": "Variables", + "typeAliasesInSubtitle": "Type Aliases", + "membersInSubtitle": "Members", + "improveThisDoc": "Improve this Doc", + "viewSource": "View Source", + "inheritance": "Inheritance", + "inheritedMembers": "Inherited Members", + "package": "Package", + "namespace": "Namespace", + "assembly": "Assembly", + "syntax": "Syntax", + "overrides": "Overrides", + "implements": "Implements", + "remarks": "Remarks", + "examples": "Examples", + "seealso": "See Also", + "declaration": "Declaration", + "parameters": "Parameters", + "typeParameters": "Type Parameters", + "type": "Type", + "name": "Name", + "description": "Description", + "returns": "Returns", + "fieldValue": "Field Value", + "propertyValue": "Property Value", + "eventType": "Event Type", + "variableValue": "Variable Value", + "typeAliasType": "Type Alias Type", + "exceptions": "Exceptions", + "condition": "Condition", + "extensionMethods": "Extension Methods", + "note": "
      Note
      ", + "warning": "
      Warning
      ", + "tip": "
      Tip
      ", + "important": "
      Important
      ", + "caution": "
      Caution
      ", + "tocToggleButton": "Show / Hide Table of Contents", + "tocFilter": "Enter here to filter...", + "search": "Search", + "searchResults": "Search Results for", + "pageFirst": "First", + "pagePrev": "Previous", + "pageNext": "Next", + "pageLast": "Last", + "inThisArticle": "In This Article", + "backToTop": "Back to top" +} diff --git a/csharp/ApiDocs/api/.gitignore b/csharp/ApiDocs/api/.gitignore new file mode 100644 index 0000000000..e8079a3bef --- /dev/null +++ b/csharp/ApiDocs/api/.gitignore @@ -0,0 +1,5 @@ +############### +# temp file # +############### +*.yml +.manifest diff --git a/csharp/ApiDocs/api/index.md b/csharp/ApiDocs/api/index.md new file mode 100644 index 0000000000..ebc1ca0e50 --- /dev/null +++ b/csharp/ApiDocs/api/index.md @@ -0,0 +1,4 @@ +# ONNX Runtime C# API Documentation + +### [Microsoft.ML.OnnxRuntime](./Microsoft.ML.OnnxRuntime.html) +### [Microsoft.ML.OnnxRuntime.Tensors](./Microsoft.ML.OnnxRuntime.Tensors.html) \ No newline at end of file diff --git a/csharp/ApiDocs/docfx.json b/csharp/ApiDocs/docfx.json new file mode 100644 index 0000000000..9f458497e5 --- /dev/null +++ b/csharp/ApiDocs/docfx.json @@ -0,0 +1,67 @@ +{ + "metadata": [ + { + "src": [ + { + "files": [ + "src/Microsoft.ML.OnnxRuntime/**.cs" + ], + "src": "../" + } + ], + "dest": "api", + "disableGitFeatures": false, + "disableDefaultFilter": false + } + ], + "build": { + "content": [ + { + "files": [ + "api/**.yml", + "api/**.html", + "api/index.md", + "api/**/index.md" + ] + }, + { + "files": [ + "toc.yml\",", + "toc.yml", + "*.md", + "api/toc.yml" + ] + } + ], + "resource": [ + { + "files": [ + //"images/**" + ] + } + ], + "overwrite": [ + { + "files": [ + "api/**.md" + ], + "exclude": [ + "obj/**", + "csharp/**" + ] + } + ], + "dest": "csharp", + "globalMetadataFiles": [], + "fileMetadataFiles": [], + "template": [ + "_exported_templates/default" + ], + "postProcessors": [], + "markdownEngineName": "markdig", + "noLangKeyword": false, + "keepFileLink": false, + "cleanupCacheHistory": false, + "disableGitFeatures": false + } +} diff --git a/csharp/ApiDocs/index.md b/csharp/ApiDocs/index.md new file mode 100644 index 0000000000..3752a7aaa6 --- /dev/null +++ b/csharp/ApiDocs/index.md @@ -0,0 +1,4 @@ +# ONNX Runtime C# Documentation + +### [Go to API Documentation](/docs/api/csharp/api/index.html) +### [Go back to ONNX Runtime Docs](https://onnxruntime.ai/docs/) diff --git a/csharp/ApiDocs/toc.yml b/csharp/ApiDocs/toc.yml new file mode 100644 index 0000000000..916f012a6d --- /dev/null +++ b/csharp/ApiDocs/toc.yml @@ -0,0 +1,3 @@ +- name: ONNX Runtime C# Api Documentation + href: api/ + homepage: api/index.md diff --git a/csharp/OnnxRuntime.CSharp.sln b/csharp/OnnxRuntime.CSharp.sln index f9f11f2e4a..ab462f498c 100644 --- a/csharp/OnnxRuntime.CSharp.sln +++ b/csharp/OnnxRuntime.CSharp.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.31613.86 +# Visual Studio Version 17 +VisualStudioVersion = 17.2.32630.192 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "OnnxRuntime", "OnnxRuntime", "{6EFBFAB8-C606-4BA4-9604-BBAF3788520D}" EndProject @@ -34,6 +34,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.OnnxRuntime.Te EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.OnnxRuntime.Tests.Devices", "test\Microsoft.ML.OnnxRuntime.Tests.Devices\Microsoft.ML.OnnxRuntime.Tests.Devices.csproj", "{30431891-3929-4394-8049-75055B92315F}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{7C7FC981-8AE7-42C6-962C-4B70D5C20C35}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApiDocs", "ApiDocs\ApiDocs.csproj", "{7D841874-815D-43C7-9A3F-DD0AD9BA9D42}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -344,6 +348,30 @@ Global {30431891-3929-4394-8049-75055B92315F}.RelWithDebInfo|iPhoneSimulator.Build.0 = Release|Any CPU {30431891-3929-4394-8049-75055B92315F}.RelWithDebInfo|x86.ActiveCfg = Debug|Any CPU {30431891-3929-4394-8049-75055B92315F}.RelWithDebInfo|x86.Build.0 = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|iPhone.ActiveCfg = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|iPhone.Build.0 = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|iPhoneSimulator.ActiveCfg = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|iPhoneSimulator.Build.0 = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|x86.ActiveCfg = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Debug|x86.Build.0 = Debug|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|Any CPU.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|iPhone.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|iPhone.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|iPhoneSimulator.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|iPhoneSimulator.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|x86.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.Release|x86.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|iPhone.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|iPhone.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|iPhoneSimulator.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|iPhoneSimulator.Build.0 = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42}.RelWithDebInfo|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -361,6 +389,7 @@ Global {19446672-EBA3-4BA6-8DFB-AB2A85AC9AA4} = {6782763B-8097-457C-AEA3-67678621DBE0} {772E0BB4-6B5A-4453-9F4A-034DCBB61B5C} = {6782763B-8097-457C-AEA3-67678621DBE0} {30431891-3929-4394-8049-75055B92315F} = {6782763B-8097-457C-AEA3-67678621DBE0} + {7D841874-815D-43C7-9A3F-DD0AD9BA9D42} = {7C7FC981-8AE7-42C6-962C-4B70D5C20C35} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C3DBDA2B-F169-4EDE-9353-858904124B75} diff --git a/csharp/sample/InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.Forms.Android/Resources/Resource.designer.cs b/csharp/sample/InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.Forms.Android/Resources/Resource.designer.cs index 79d5bc2c84..7a21331403 100644 --- a/csharp/sample/InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.Forms.Android/Resources/Resource.designer.cs +++ b/csharp/sample/InferenceSample/Microsoft.ML.OnnxRuntime.InferenceSample.Forms.Android/Resources/Resource.designer.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.OnnxRuntime.InferenceSample.Forms.Droid { - + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Xamarin.Android.Build.Tasks", "1.0.0.0")] public partial class Resource { diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.shared.cs index 027cbfdc78..98e1833aed 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.shared.cs @@ -773,14 +773,14 @@ namespace Microsoft.ML.OnnxRuntime if (prepackedWeightsContainer == null) { - NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, NativeMethods.GetPlatformSerializedString(modelPath), + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, NativeOnnxValueHelper.GetPlatformSerializedString(modelPath), options.Handle, out session)); } else { NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSessionWithPrepackedWeightsContainer( - envHandle, NativeMethods.GetPlatformSerializedString(modelPath), + envHandle, NativeOnnxValueHelper.GetPlatformSerializedString(modelPath), options.Handle, prepackedWeightsContainer.Pointer, out session)); } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj index 661a9e8cba..679361c336 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj @@ -72,6 +72,9 @@ false portable + + false + Microsoft.ML.OnnxRuntime.Managed @@ -176,6 +179,14 @@ $(OrtConstants);__XAMARIN__ + + + $(OrtConstants);__ENABLE_TRAINING_ON_DEVICE__ + + $(DefineConstants);$(OrtConstants) @@ -336,4 +347,4 @@ /> - \ No newline at end of file + diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs index f008d2fbff..c576c8d38a 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs @@ -248,6 +248,10 @@ namespace Microsoft.ML.OnnxRuntime public IntPtr InvokeOp; public IntPtr ReleaseOp; public IntPtr SessionOptionsAppendExecutionProvider; + public IntPtr CopyKernelInfo; + public IntPtr ReleaseKernelInfo; + + public IntPtr GetTrainingApi; } internal static class NativeMethods @@ -421,7 +425,7 @@ namespace Microsoft.ML.OnnxRuntime OrtReleaseCUDAProviderOptions = (DOrtReleaseCUDAProviderOptions)Marshal.GetDelegateForFunctionPointer(api_.ReleaseCUDAProviderOptions, typeof(DOrtReleaseCUDAProviderOptions)); SessionOptionsAppendExecutionProvider = (DSessionOptionsAppendExecutionProvider)Marshal.GetDelegateForFunctionPointer( - api_.SessionOptionsAppendExecutionProvider, + api_.SessionOptionsAppendExecutionProvider, typeof(DSessionOptionsAppendExecutionProvider)); } @@ -443,7 +447,7 @@ namespace Microsoft.ML.OnnxRuntime [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)] public static extern ref OrtApiBase OrtGetApiBase(); - #region Runtime/Environment API +#region Runtime/Environment API [UnmanagedFunctionPointer(CallingConvention.Winapi)] public delegate IntPtr /* OrtStatus* */DOrtCreateEnv(LogLevel default_warning_level, string logId, out IntPtr /*(OrtEnv*)*/ env); @@ -462,9 +466,9 @@ namespace Microsoft.ML.OnnxRuntime public delegate IntPtr /* OrtStatus* */DOrtDisableTelemetryEvents(IntPtr /*(OrtEnv*)*/ env); public static DOrtDisableTelemetryEvents OrtDisableTelemetryEvents; - #endregion Runtime/Environment API +#endregion Runtime/Environment API - #region Provider Options API +#region Provider Options API /// /// Creates native OrtTensorRTProviderOptions instance @@ -553,9 +557,9 @@ namespace Microsoft.ML.OnnxRuntime [UnmanagedFunctionPointer(CallingConvention.Winapi)] public delegate void DOrtReleaseCUDAProviderOptions(IntPtr /*(OrtCUDAProviderOptions*)*/ cudaProviderOptionsInstance); public static DOrtReleaseCUDAProviderOptions OrtReleaseCUDAProviderOptions; - #endregion +#endregion - #region Status API +#region Status API [UnmanagedFunctionPointer(CallingConvention.Winapi)] public delegate ErrorCode DOrtGetErrorCode(IntPtr /*(OrtStatus*)*/status); public static DOrtGetErrorCode OrtGetErrorCode; @@ -570,9 +574,9 @@ namespace Microsoft.ML.OnnxRuntime public delegate void DOrtReleaseStatus(IntPtr /*(OrtStatus*)*/ statusPtr); public static DOrtReleaseStatus OrtReleaseStatus; - #endregion Status API +#endregion Status API - #region InferenceSession API +#region InferenceSession API [UnmanagedFunctionPointer(CallingConvention.Winapi)] public delegate IntPtr /* OrtStatus* */DOrtCreateSession( @@ -589,7 +593,7 @@ namespace Microsoft.ML.OnnxRuntime /// /// Native OrtEnv instance /// UTF-8 bytes corresponding to model string path - /// Native SessionOptions instance + /// Native SessionOptions instance /// Native OrtPrepackedWeightsContainer instance /// (Output) Created native OrtSession instance [UnmanagedFunctionPointer(CallingConvention.Winapi)] @@ -617,7 +621,7 @@ namespace Microsoft.ML.OnnxRuntime /// Native OrtEnv instance /// Byte array correspoonding to the model /// Size of the model in bytes - /// Native SessionOptions instance + /// Native SessionOptions instance /// Native OrtPrepackedWeightsContainer instance /// (Output) Created native OrtSession instance [UnmanagedFunctionPointer(CallingConvention.Winapi)] @@ -748,9 +752,9 @@ namespace Microsoft.ML.OnnxRuntime out UIntPtr /*(ulong* out)*/ startTime); public static DOrtSessionGetProfilingStartTimeNs OrtSessionGetProfilingStartTimeNs; - #endregion InferenceSession API +#endregion InferenceSession API - #region SessionOptions API +#region SessionOptions API [UnmanagedFunctionPointer(CallingConvention.Winapi)] public delegate IntPtr /*(OrtStatus*)*/ DOrtCreateSessionOptions(out IntPtr /*(OrtSessionOptions**)*/ sessionOptions); @@ -991,9 +995,9 @@ namespace Microsoft.ML.OnnxRuntime /// /// Append an execution provider instance to the native OrtSessionOptions instance. - /// + /// /// 'SNPE' and 'XNNPACK' are currently supported as providerName values. - /// + /// /// The number of providerOptionsKeys must match the number of providerOptionsValues and equal numKeys. /// /// Native OrtSessionOptions instance @@ -1711,13 +1715,5 @@ namespace Microsoft.ML.OnnxRuntime public static DOrtReleasePrepackedWeightsContainer OrtReleasePrepackedWeightsContainer; #endregion - - public static byte[] GetPlatformSerializedString(string str) - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - return System.Text.Encoding.Unicode.GetBytes(str + Char.MinValue); - else - return System.Text.Encoding.UTF8.GetBytes(str + Char.MinValue); - } } //class NativeMethods } //namespace diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.shared.cs index 67781b82f5..feff70782f 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.shared.cs @@ -102,6 +102,20 @@ namespace Microsoft.ML.OnnxRuntime return result; } + /// + /// Converts C# UTF-16 string to UTF-8 zero terminated + /// byte[] instance + /// + /// string to be converted + /// UTF-8 encoded equivalent + internal static byte[] GetPlatformSerializedString(string str) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return System.Text.Encoding.Unicode.GetBytes(str + Char.MinValue); + else + return StringToZeroTerminatedUtf8(str); + } + // Delegate for string extraction from an arbitrary input/output object internal delegate string NameExtractor(TInput input); } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.shared.cs index c185b218e5..61a3eb680f 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.shared.cs @@ -73,7 +73,7 @@ namespace Microsoft.ML.OnnxRuntime /// Exception caching: May throw an exception on every call, if the `OrtEnv` constructor threw an exception /// during lazy initialization /// - internal static IntPtr Handle + internal static IntPtr Handle { get { @@ -178,4 +178,4 @@ namespace Microsoft.ML.OnnxRuntime } #endregion } -} \ No newline at end of file +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.shared.cs index 5e6f1b8ec3..a65839b183 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/ProviderOptions.shared.cs @@ -247,7 +247,7 @@ namespace Microsoft.ML.OnnxRuntime /// /// CoreML flags for use with SessionOptions /// - /// + /// [Flags] public enum CoreMLFlags : uint { @@ -261,7 +261,7 @@ namespace Microsoft.ML.OnnxRuntime /// /// NNAPI flags for use with SessionOptions /// - /// + /// [Flags] public enum NnapiFlags { diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs index 944ded201a..083dc1f1f8 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs @@ -11,7 +11,7 @@ namespace Microsoft.ML.OnnxRuntime { /// /// Graph optimization level to use with SessionOptions - /// [https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md] + /// [https://github.com/microsoft/onnxruntime/blob/main/docs/ONNX_Runtime_Graph_Optimizations.md] /// public enum GraphOptimizationLevel { @@ -408,13 +408,13 @@ namespace Microsoft.ML.OnnxRuntime { providerOptions = new Dictionary(); } - + var keysArray = NativeOnnxValueHelper.ConvertNamesToUtf8( providerOptions.Keys.ToArray(), n => n, cleanupList); var valuesArray = NativeOnnxValueHelper.ConvertNamesToUtf8( providerOptions.Values.ToArray(), n => n, cleanupList); - + NativeApiStatus.VerifySuccess(NativeMethods.SessionOptionsAppendExecutionProvider( handle, epArray[0], keysArray, valuesArray, (UIntPtr)providerOptions.Count)); } @@ -426,7 +426,7 @@ namespace Microsoft.ML.OnnxRuntime /// (Deprecated) Loads a DLL named 'libraryPath' and looks for this entry point: /// OrtStatus* RegisterCustomOps(OrtSessionOptions* options, const OrtApiBase* api); /// It then passes in the provided session options to this function along with the api base. - /// Deprecated in favor of RegisterCustomOpLibraryV2() because it provides users with the library handle + /// Deprecated in favor of RegisterCustomOpLibraryV2() because it provides users with the library handle /// to release when all sessions relying on it are destroyed /// /// path to the custom op library @@ -589,7 +589,7 @@ namespace Microsoft.ML.OnnxRuntime { if (!_enableProfiling && value) { - NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableProfiling(handle, NativeMethods.GetPlatformSerializedString(ProfileOutputPathPrefix))); + NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableProfiling(handle, NativeOnnxValueHelper.GetPlatformSerializedString(ProfileOutputPathPrefix))); _enableProfiling = true; } else if (_enableProfiling && !value) @@ -615,7 +615,7 @@ namespace Microsoft.ML.OnnxRuntime { if (value != _optimizedModelFilePath) { - NativeApiStatus.VerifySuccess(NativeMethods.OrtSetOptimizedModelFilePath(handle, NativeMethods.GetPlatformSerializedString(value))); + NativeApiStatus.VerifySuccess(NativeMethods.OrtSetOptimizedModelFilePath(handle, NativeOnnxValueHelper.GetPlatformSerializedString(value))); _optimizedModelFilePath = value; } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs new file mode 100644 index 0000000000..4cff5bccb1 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/CheckpointState.shared.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ +#if __ENABLE_TRAINING_ON_DEVICE__ + /// + /// Holds the Checkpoint State as generated/consumed by on-device training APIs + /// + public class CheckpointState : SafeHandle + { + internal IntPtr Handle + { + get + { + return handle; + } + } + + /// + /// Creates CheckpointState by loading state from path. + /// absolute path to checkpoint file. + /// + public CheckpointState(string checkpointPath) + : base(IntPtr.Zero, true) + { + if (NativeTrainingMethods.TrainingEnabled()) + { + var envHandle = OrtEnv.Handle; // just so it is initialized + LoadCheckpoint(checkpointPath); + } + else + { + throw new InvalidOperationException("Training is disabled in the current build"); + } + } + + /// + /// Overrides SafeHandle.IsInvalid + /// + /// returns true if handle is equal to Zero + public override bool IsInvalid { get { return handle == IntPtr.Zero; } } + + /// + /// Loads Checkpoint state from path + /// + /// absolute path to checkpoint + private void LoadCheckpoint(string checkpointPath) + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtLoadCheckpoint(NativeOnnxValueHelper.GetPlatformSerializedString(checkpointPath), out handle)); + } + +#region SafeHandle + /// + /// Overrides SafeHandle.ReleaseHandle() to properly dispose of + /// the native instance of CheckpointState + /// + /// always returns true + protected override bool ReleaseHandle() + { + NativeTrainingMethods.OrtReleaseCheckpointState(handle); + handle = IntPtr.Zero; + return true; + } + +#endregion + } +#endif +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/NativeTrainingMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/NativeTrainingMethods.shared.cs new file mode 100644 index 0000000000..445662151f --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/NativeTrainingMethods.shared.cs @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ +#if __ENABLE_TRAINING_ON_DEVICE__ + // OrtTrainingApi (onnxruntime_training_c_api.cc) + [StructLayout(LayoutKind.Sequential)] + public struct OrtTrainingApi + { + public IntPtr LoadCheckpoint; + public IntPtr SaveCheckpoint; + public IntPtr CreateTrainingSession; + public IntPtr TrainingSessionGetTrainingModelOutputCount; + public IntPtr TrainingSessionGetEvalModelOutputCount; + public IntPtr TrainingSessionGetTrainingModelOutputName; + public IntPtr TrainingSessionGetEvalModelOutputName; + public IntPtr ResetGrad; + public IntPtr TrainStep; + public IntPtr EvalStep; + public IntPtr SetLearningRate; + public IntPtr GetLearningRate; + public IntPtr OptimizerStep; + public IntPtr RegisterLinearLRScheduler; + public IntPtr SchedulerStep; + public IntPtr ReleaseTrainingSession; + public IntPtr ReleaseCheckpointState; + } + + internal static class NativeTrainingMethods + { + static OrtApi api_; + static OrtTrainingApi trainingApi_; + static IntPtr trainingApiPtr; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate ref OrtApi DOrtGetApi(UInt32 version); + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /* OrtTrainingApi* */ DOrtGetTrainingApi(UInt32 version); + public static DOrtGetTrainingApi OrtGetTrainingApi; + + static NativeTrainingMethods() + { + DOrtGetApi OrtGetApi = (DOrtGetApi)Marshal.GetDelegateForFunctionPointer(NativeMethods.OrtGetApiBase().GetApi, typeof(DOrtGetApi)); + + // TODO: Make this save the pointer, and not copy the whole structure across + api_ = (OrtApi)OrtGetApi(4 /*ORT_API_VERSION*/); + + OrtGetTrainingApi = (DOrtGetTrainingApi)Marshal.GetDelegateForFunctionPointer(api_.GetTrainingApi, typeof(DOrtGetTrainingApi)); + trainingApiPtr = OrtGetTrainingApi(4 /*ORT_API_VERSION*/); + if (trainingApiPtr != IntPtr.Zero) + { + trainingApi_ = (OrtTrainingApi)Marshal.PtrToStructure(trainingApiPtr, typeof(OrtTrainingApi)); + OrtLoadCheckpoint = (DOrtLoadCheckpoint)Marshal.GetDelegateForFunctionPointer(trainingApi_.LoadCheckpoint, typeof(DOrtLoadCheckpoint)); + OrtSaveCheckpoint = (DOrtSaveCheckpoint)Marshal.GetDelegateForFunctionPointer(trainingApi_.SaveCheckpoint, typeof(DOrtSaveCheckpoint)); + OrtCreateTrainingSession = (DOrtCreateTrainingSession)Marshal.GetDelegateForFunctionPointer(trainingApi_.CreateTrainingSession, typeof(DOrtCreateTrainingSession)); + OrtGetTrainingModelOutputCount = (DOrtGetTrainingModelOutputCount)Marshal.GetDelegateForFunctionPointer(trainingApi_.TrainingSessionGetTrainingModelOutputCount, typeof(DOrtGetTrainingModelOutputCount)); + OrtGetEvalModelOutputCount = (DOrtGetEvalModelOutputCount)Marshal.GetDelegateForFunctionPointer(trainingApi_.TrainingSessionGetEvalModelOutputCount, typeof(DOrtGetEvalModelOutputCount)); + OrtGetTrainingModelOutputName = (DOrtGetTrainingModelOutputName)Marshal.GetDelegateForFunctionPointer(trainingApi_.TrainingSessionGetTrainingModelOutputName, typeof(DOrtGetTrainingModelOutputName)); + OrtGetEvalModelOutputName = (DOrtGetEvalModelOutputName)Marshal.GetDelegateForFunctionPointer(trainingApi_.TrainingSessionGetEvalModelOutputName, typeof(DOrtGetEvalModelOutputName)); + OrtResetGrad = (DOrtResetGrad)Marshal.GetDelegateForFunctionPointer(trainingApi_.ResetGrad, typeof(DOrtResetGrad)); + OrtTrainStep = (DOrtTrainStep)Marshal.GetDelegateForFunctionPointer(trainingApi_.TrainStep, typeof(DOrtTrainStep)); + OrtEvalStep = (DOrtEvalStep)Marshal.GetDelegateForFunctionPointer(trainingApi_.EvalStep, typeof(DOrtEvalStep)); + OrtSetLearningRate = (DOrtSetLearningRate)Marshal.GetDelegateForFunctionPointer(trainingApi_.SetLearningRate, typeof(DOrtSetLearningRate)); + OrtGetLearningRate = (DOrtGetLearningRate)Marshal.GetDelegateForFunctionPointer(trainingApi_.GetLearningRate, typeof(DOrtGetLearningRate)); + OrtOptimizerStep = (DOrtOptimizerStep)Marshal.GetDelegateForFunctionPointer(trainingApi_.OptimizerStep, typeof(DOrtOptimizerStep)); + OrtRegisterLinearLRScheduler = (DOrtRegisterLinearLRScheduler)Marshal.GetDelegateForFunctionPointer(trainingApi_.RegisterLinearLRScheduler, typeof(DOrtRegisterLinearLRScheduler)); + OrtSchedulerStep = (DOrtSchedulerStep)Marshal.GetDelegateForFunctionPointer(trainingApi_.SchedulerStep, typeof(DOrtSchedulerStep)); + OrtReleaseTrainingSession = (DOrtReleaseTrainingSession)Marshal.GetDelegateForFunctionPointer(trainingApi_.ReleaseTrainingSession, typeof(DOrtReleaseTrainingSession)); + OrtReleaseCheckpointState = (DOrtReleaseCheckpointState)Marshal.GetDelegateForFunctionPointer(trainingApi_.ReleaseCheckpointState, typeof(DOrtReleaseCheckpointState)); + } + + } + +#region TrainingSession API + + /// + /// Creates an instance of OrtSession with provided parameters + /// + /// checkpoint string path + /// (Output) Loaded OrtCheckpointState instance + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /* OrtStatus* */DOrtLoadCheckpoint( + byte[] checkpointPath, + out IntPtr /* (OrtCheckpointState**) */ checkpointState); + + public static DOrtLoadCheckpoint OrtLoadCheckpoint; + + /// + /// Creates an instance of OrtSession with provided parameters + /// + /// checkpoint string path + /// (Output) Loaded OrtCheckpointState instance + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /* OrtStatus* */DOrtSaveCheckpoint( + byte[] checkpointPath, + IntPtr /*(OrtTrainingSession*)*/ session, + bool saveOptimizerState); + + public static DOrtSaveCheckpoint OrtSaveCheckpoint; + + /// + /// Creates an instance of OrtSession with provided parameters + /// + /// Native OrtEnv instance + /// Native SessionOptions instance + /// Loaded OrtCheckpointState instance + /// model string path + /// model string path + /// model string path + /// (Output) Created native OrtTrainingSession instance + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /* OrtStatus* */DOrtCreateTrainingSession( + IntPtr /* (OrtEnv*) */ environment, + IntPtr /* (OrtSessionOptions*) */ sessionOptions, + IntPtr /* (OrtCheckpointState*) */ checkpointState, + byte[] trainModelPath, + byte[] evalModelPath, + byte[] optimizerModelPath, + out IntPtr /* (OrtTrainingSession**) */ session); + + public static DOrtCreateTrainingSession OrtCreateTrainingSession; + + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/ DOrtGetTrainingModelOutputCount( + IntPtr /*(OrtSession*)*/ session, + out UIntPtr count); + + public static DOrtGetTrainingModelOutputCount OrtGetTrainingModelOutputCount; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/ DOrtGetEvalModelOutputCount( + IntPtr /*(OrtSession*)*/ session, + out UIntPtr count); + + public static DOrtGetEvalModelOutputCount OrtGetEvalModelOutputCount; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/ DOrtGetTrainingModelOutputName( + IntPtr /*(OrtSession*)*/ session, + UIntPtr index, + IntPtr /*(OrtAllocator*)*/ allocator, + out IntPtr /*(char**)*/name); + + public static DOrtGetTrainingModelOutputName OrtGetTrainingModelOutputName; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/ DOrtGetEvalModelOutputName( + IntPtr /*(OrtSession*)*/ session, + UIntPtr index, + IntPtr /*(OrtAllocator*)*/ allocator, + out IntPtr /*(char**)*/name); + + public static DOrtGetEvalModelOutputName OrtGetEvalModelOutputName; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(OrtStatus*)*/ DOrtResetGrad( + IntPtr /*(OrtSession*)*/ session); + + public static DOrtResetGrad OrtResetGrad; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtTrainStep( + IntPtr /*(OrtTrainingSession*)*/ session, + IntPtr /*(OrtSessionRunOptions*)*/ runOptions, // can be null to use the default options + UIntPtr inputCount, + IntPtr[] /* (OrtValue*[])*/ inputValues, + UIntPtr outputCount, + IntPtr[] outputValues /* An array of output value pointers. Array must be allocated by the caller */ + ); + + public static DOrtTrainStep OrtTrainStep; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtEvalStep( + IntPtr /*(OrtTrainingSession*)*/ session, + IntPtr /*(OrtSessionRunOptions*)*/ runOptions, // can be null to use the default options + UIntPtr inputCount, + IntPtr[] /* (OrtValue*[])*/ inputValues, + UIntPtr outputCount, + IntPtr[] outputValues /* An array of output value pointers. Array must be allocated by the caller */ + ); + + public static DOrtEvalStep OrtEvalStep; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtOptimizerStep( + IntPtr /*(OrtTrainingSession*)*/ session, + IntPtr /*(OrtSessionRunOptions*)*/ runOptions // can be null to use the default options + ); + + public static DOrtOptimizerStep OrtOptimizerStep; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtSetLearningRate( + IntPtr /*(OrtTrainingSession*)*/ session, + float learningRate + ); + + public static DOrtSetLearningRate OrtSetLearningRate; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtGetLearningRate( + IntPtr /*(OrtTrainingSession*)*/ session, + out float learningRate + ); + + public static DOrtGetLearningRate OrtGetLearningRate; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtRegisterLinearLRScheduler( + IntPtr /*(OrtTrainingSession*)*/ session, + long warmupStepCount, + long totalStepCount, + float learningRate + ); + public static DOrtRegisterLinearLRScheduler OrtRegisterLinearLRScheduler; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate IntPtr /*(ONNStatus*)*/ DOrtSchedulerStep( + IntPtr /*(OrtTrainingSession*)*/ session + ); + public static DOrtSchedulerStep OrtSchedulerStep; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate void DOrtReleaseTrainingSession(IntPtr /*(OrtSession*)*/session); + public static DOrtReleaseTrainingSession OrtReleaseTrainingSession; + + [UnmanagedFunctionPointer(CallingConvention.Winapi)] + public delegate void DOrtReleaseCheckpointState(IntPtr /*(OrtSession*)*/session); + public static DOrtReleaseCheckpointState OrtReleaseCheckpointState; + +#endregion TrainingSession API + + public static bool TrainingEnabled() + { + if (trainingApiPtr == IntPtr.Zero) + { + return false; + } + return true; + } + } //class NativeTrainingMethods +#endif +} //namespace diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs new file mode 100644 index 0000000000..b5b5a7c598 --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/Training/TrainingSession.shared.cs @@ -0,0 +1,504 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ +#if __ENABLE_TRAINING_ON_DEVICE__ + enum LRScheduler + { + None = 0, + Constant = 1, + Linear = 2 + } + /// + /// Represents a Training Session on an ONNX Model. + /// This is a IDisposable class and it must be disposed of + /// using either a explicit call to Dispose() method or + /// a pattern of using() block. If this is a member of another + /// class that class must also become IDisposable and it must + /// dispose of TrainingSession in its Dispose() method. + /// + public class TrainingSession : IDisposable + { + /// + /// A pointer to a underlying native instance of OrtTrainingSession + /// + private IntPtr _nativeHandle; + + private ulong _trainOutputCount; + private ulong _evalOutputCount; + private List _trainOutputNames; + private List _evalOutputNames; + + private SessionOptions _builtInSessionOptions = null; + private RunOptions _builtInRunOptions = null; + private LRScheduler _scheduler = LRScheduler.None; + private bool _disposed = false; + + #region Public API + + /// + /// Creates TrainingSession from the model and checkpoint in . + /// + /// Model checkpoint loaded into . + /// Specify path to training model graph. + /// Specify path to eval model graph. + /// Specify path to optimizer model graph. + public TrainingSession(CheckpointState state, string trainModelPath, string evalModelPath, string optimizerModelPath) + { + Init(null, state, NativeOnnxValueHelper.GetPlatformSerializedString(trainModelPath), NativeOnnxValueHelper.GetPlatformSerializedString(evalModelPath), NativeOnnxValueHelper.GetPlatformSerializedString(optimizerModelPath)); + } + + /// + /// Creates TrainingSession from the model and checkpoint in . + /// + /// Model checkpoint loaded into . + /// Specify path to training model graph. + /// Specify path to optimizer model graph. + public TrainingSession(CheckpointState state, string trainModelPath, string optimizerModelPath) + { + Init(null, state, NativeOnnxValueHelper.GetPlatformSerializedString(trainModelPath), null, NativeOnnxValueHelper.GetPlatformSerializedString(optimizerModelPath)); + } + + /// + /// Creates TrainingSession from the model and checkpoint in . + /// + /// Model checkpoint loaded into . + /// Specify path to training model graph. + public TrainingSession(CheckpointState state, string trainModelPath) + { + Init(null, state, NativeOnnxValueHelper.GetPlatformSerializedString(trainModelPath), null, null); + } + + + /// + /// Creates TrainingSession from the model and checkpoint in . + /// + /// Session options + /// Model checkpoint loaded into . + /// Specify path to training model graph. + /// Specify path to eval model graph. + /// Specify path to optimizer model graph. + public TrainingSession(SessionOptions options, CheckpointState state, string trainModelPath, string evalModelPath, string optimizerModelPath) + { + Init(options, state, NativeOnnxValueHelper.GetPlatformSerializedString(trainModelPath), NativeOnnxValueHelper.GetPlatformSerializedString(evalModelPath), NativeOnnxValueHelper.GetPlatformSerializedString(optimizerModelPath)); + } + + /// + /// Runs a train step on the loaded model for the given inputs. + /// + /// Specify a collection of that indicates the input values. + /// Specify a collection of that indicates the output values. + public void TrainStep( + IReadOnlyCollection inputValues, + IReadOnlyCollection outputValues) + { + TrainStep(_builtInRunOptions, inputValues, outputValues); + } + + /// + /// Runs a train step on the loaded model for the given inputs. Uses the given RunOptions for this run. + /// + /// Specify for step. + /// Specify a collection of that indicates the input values. + /// Specify a collection of that indicates the output values. + public void TrainStep( + RunOptions options, + IReadOnlyCollection inputValues, + IReadOnlyCollection outputValues) + { + if (_trainOutputCount!= (ulong)outputValues.Count()) + { + throw new ArgumentException($"Length of {nameof(outputValues)} ({outputValues.Count}) must match that of train model ({_trainOutputCount})."); + } + IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true); + + IntPtr[] outputValuesArray = GetOrtValuesHandles(outputValues, false); /* pointers to Pre-allocated OrtValue instances */ + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtTrainStep(_nativeHandle, options.Handle, (UIntPtr)inputValues.Count, + inputValuesArray, (UIntPtr)outputValues.Count, outputValuesArray)); + } + + /// + /// Runs the loaded model for the given inputs, and fetches the graph outputs. + /// + /// Specify a collection of that indicates the input values. + /// Output Tensors in a Collection of NamedOnnxValue. User must dispose the output. + public IDisposableReadOnlyCollection TrainStep( + IReadOnlyCollection inputValues) + { + using (var ortValues = new DisposableList((int)_trainOutputCount)) + { + IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true); + IntPtr[] outputValuesArray = new IntPtr[(int)_trainOutputCount]; + + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtTrainStep(_nativeHandle, _builtInRunOptions.Handle, (UIntPtr)inputValues.Count, + inputValuesArray, (UIntPtr)_trainOutputCount, outputValuesArray)); + foreach (var v in outputValuesArray) + { + ortValues.Add(new OrtValue(v)); + } + + var result = new DisposableList(_trainOutputNames.Count); + try + { + for (int i = 0; i < ortValues.Count; i++) + { + var ortValue = ortValues[i]; + result.Add(DisposableNamedOnnxValue.CreateFromOrtValue(_trainOutputNames[i], ortValue)); + } + } + catch (OnnxRuntimeException) + { + result.Dispose(); + throw; + } + return result; + } + } + + /// + /// Runs the loaded model for the given inputs, and fetches the specified outputs in . Uses the given RunOptions for this run. + /// + /// Specify for step. + /// Specify a collection of that indicates the input values. + /// Output Tensors in a Collection of NamedOnnxValue. User must dispose the output. + public IDisposableReadOnlyCollection TrainStep( + RunOptions options, + IReadOnlyCollection inputValues) + { + using (var ortValues = new DisposableList((int)_trainOutputCount)) + { + IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true); + IntPtr[] outputValuesArray = new IntPtr[(int)_trainOutputCount]; + + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtTrainStep(_nativeHandle, options.Handle, (UIntPtr)inputValues.Count, + inputValuesArray, (UIntPtr)_trainOutputCount, outputValuesArray)); + foreach (var v in outputValuesArray) + { + ortValues.Add(new OrtValue(v)); + } + + var result = new DisposableList(_trainOutputNames.Count); + try + { + for (int i = 0; i < ortValues.Count; i++) + { + var ortValue = ortValues[i]; + result.Add(DisposableNamedOnnxValue.CreateFromOrtValue(_trainOutputNames[i], ortValue)); + } + } + catch (OnnxRuntimeException) + { + result.Dispose(); + throw; + } + return result; + } + } + + /// + /// Sets the reset grad flag on the training graph. The gradient buffers will be reset while executing the + /// next train step. + /// + public void ResetGrad() + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtResetGrad(_nativeHandle)); + } + + /// + /// Runs an eval step on the loaded model for the given inputs. The eval graph must be passed while TrainingSession creation. + /// + /// Specify a collection of that indicates the input values. + /// Specify a collection of that indicates the output values. + public void EvalStep( + IReadOnlyCollection inputValues, + IReadOnlyCollection outputValues) + { + EvalStep(_builtInRunOptions, inputValues, outputValues); + } + + /// + /// Runs an eval step on the loaded model for the given inputs. The eval graph must be passed while TrainingSession creation. + /// + /// Specify for step. + /// Specify a collection of that indicates the input values. + /// Specify a collection of that indicates the output values. + public void EvalStep( + RunOptions options, + IReadOnlyCollection inputValues, + IReadOnlyCollection outputValues) + { + if (!_evalOutputCount.Equals(outputValues.Count)) + { + throw new ArgumentException($"Length of {nameof(outputValues)} ({outputValues.Count}) must match that of train model ({_trainOutputCount})."); + } + IntPtr[] inputValuesArray = GetOrtValuesHandles(inputValues, true); + + IntPtr[] outputValuesArray = GetOrtValuesHandles(outputValues, false); /* pointers to Pre-allocated OrtValue instances */ + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtTrainStep(_nativeHandle, options.Handle, (UIntPtr)inputValues.Count, + inputValuesArray, (UIntPtr)outputValues.Count, outputValuesArray)); + } + + + /// + /// Sets a constant learning rate for the session. LR must be controlled by either this method + /// or by registering a LR scheduler. + /// + public void SetLearningRate(float learningRate) + { + if (_scheduler != LRScheduler.None && _scheduler != LRScheduler.Constant) + { + throw new InvalidOperationException("Cannot set constant LR while using LR scheduler."); + } + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtSetLearningRate(_nativeHandle, learningRate)); + _scheduler = LRScheduler.Constant; + } + + /// + /// Gets the current learning rate for the session. + /// + public float GetLearningRate() + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtGetLearningRate(_nativeHandle, out float lr)); + return lr; + } + + /// + /// Registers a linear learning rate scheduler for the session. LR must be controlled by either + /// the SetLearningRate method or by registering a LR scheduler. + /// Number of warmup steps + /// Number of total steps + /// Initial learning rate + /// + public void RegisterLinearLRScheduler(long warmupStepCount, + long totalStepCount, + float initialLearningRate) + { + if (_scheduler != LRScheduler.None && _scheduler != LRScheduler.Constant) + { + throw new InvalidOperationException("Cannot set LR scheduler while using constant LR."); + } + + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtRegisterLinearLRScheduler(_nativeHandle, warmupStepCount,totalStepCount, initialLearningRate)); + _scheduler = LRScheduler.Linear; + } + + /// + /// Runs a LR scheduler step. There must be a valid LR scheduler registered for the training session. + /// + public void SchedulerStep() + { + if (_scheduler == LRScheduler.Constant || _scheduler == LRScheduler.None) + { + throw new InvalidOperationException("Cannot take scheduler step without registering a valid LR scheduler."); + } + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtSchedulerStep(_nativeHandle)); + } + + /// + /// Runs an optimizer step on the loaded model for the given inputs. The optimizer graph must be passed while TrainingSession creation. + /// + public void OptimizerStep() + { + OptimizerStep(_builtInRunOptions); + } + + /// + /// Runs an eval step on the loaded model for the given inputs. The eval graph must be passed while TrainingSession creation. + /// + /// Specify for step. + /// Specify a collection of that indicates the output values. + public void OptimizerStep(RunOptions options) + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtOptimizerStep(_nativeHandle, options.Handle)); + + } + + /// + /// Saves a checkpoint to path. It can be loaded into + /// + /// Specify path for saving the checkpoint. + /// SFlag indicating whether to save optimizer state or not. + public void SaveCheckpoint(string path, bool saveOptimizerState = false) + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtSaveCheckpoint(NativeOnnxValueHelper.GetPlatformSerializedString(path), _nativeHandle, saveOptimizerState)); + } + + #endregion + #region private methods + + private void Init(SessionOptions sessOptions, CheckpointState state, byte[] trainModelPath, byte[] evalModelPath, byte[] optimizerModelPath) + { + if (!NativeTrainingMethods.TrainingEnabled()) + { + throw new InvalidOperationException("Training is disabled in the current build."); + } + var options = sessOptions; + if (sessOptions == null) + { + _builtInSessionOptions = new SessionOptions(); + options = _builtInSessionOptions; + } + var envHandle = OrtEnv.Handle; + try + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtCreateTrainingSession(envHandle, options.Handle, state.Handle, trainModelPath, + evalModelPath, optimizerModelPath, out _nativeHandle)); + + UIntPtr outputCount = UIntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtGetTrainingModelOutputCount(_nativeHandle, out outputCount)); + _trainOutputCount = outputCount.ToUInt64(); + + // get all the output names and metadata + _trainOutputNames = new List(); + for (ulong i = 0; i < _trainOutputCount; i++) + { + _trainOutputNames.Add(GetOutputName(i, true)); + } + + if (evalModelPath != null) + { + outputCount = UIntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtGetEvalModelOutputCount(_nativeHandle, out outputCount)); + _evalOutputCount = outputCount.ToUInt64(); + _evalOutputNames = new List(); + for (ulong i = 0; i < _evalOutputCount; i++) + { + _evalOutputNames.Add(GetOutputName(i, false)); + } + } + + _builtInRunOptions = new RunOptions(); // create a default built-in run option, and avoid creating a new one every run() call + } + catch (Exception) + { + CleanupHelper(true); + throw; + } + } + + private string GetOutputName(ulong index, bool training) + { + var allocator = OrtAllocator.DefaultInstance; + IntPtr nameHandle; + string str = null; + if (training) + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtGetTrainingModelOutputName( + _nativeHandle, + (UIntPtr)index, + allocator.Pointer, + out nameHandle)); + } + else + { + NativeApiStatus.VerifySuccess(NativeTrainingMethods.OrtGetEvalModelOutputName( + _nativeHandle, + (UIntPtr)index, + allocator.Pointer, + out nameHandle)); + } + + using (var ortAllocation = new OrtMemoryAllocation(allocator, nameHandle, 0)) + { + str = NativeOnnxValueHelper.StringFromNativeUtf8(nameHandle); + } + + return str; + } + + private IntPtr[] GetOrtValuesHandles(IReadOnlyCollection values, bool input) + { + var valuesArray = new IntPtr[values.Count]; + for (int index = 0; index < values.Count; ++index) + { + var v = values.ElementAt(index); + if (!input && v.ElementType == Tensors.TensorElementType.String) + { + throw new NotSupportedException("Using string type FixedBufferOnnxValue in outputs is not supported."); + } + valuesArray[index] = v.Value.Handle; + } + return valuesArray; + } + + /// + /// Other classes access + /// + internal IntPtr Handle + { + get + { + return _nativeHandle; + } + } + + #endregion + + #region IDisposable + + /// + /// Finalizer. + /// + ~TrainingSession() + { + Dispose(false); + } + + /// + /// IDisposable implementation + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// IDisposable implementation + /// + /// true if invoked from Dispose() method + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + CleanupHelper(disposing); + _disposed = true; + } + + private void CleanupHelper(bool disposing) + { + if (disposing) + { + if (_builtInRunOptions != null) + { + _builtInRunOptions.Dispose(); + _builtInRunOptions = null; + } + + if (_builtInSessionOptions != null) + { + _builtInSessionOptions.Dispose(); + _builtInSessionOptions = null; + } + } + + // cleanup unmanaged resources + if (_nativeHandle != IntPtr.Zero) + { + NativeTrainingMethods.OrtReleaseTrainingSession(_nativeHandle); + _nativeHandle = IntPtr.Zero; + } + } + + #endregion + } +#endif +} diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/targets/net6.0-ios/targets.xml b/csharp/src/Microsoft.ML.OnnxRuntime/targets/net6.0-ios/targets.xml index 02043bcad2..3eb9720af5 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/targets/net6.0-ios/targets.xml +++ b/csharp/src/Microsoft.ML.OnnxRuntime/targets/net6.0-ios/targets.xml @@ -1,16 +1,8 @@ - - Framework - True - True - True - -lc++ - CoreML - - - Framework + + Static True True True diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/targets/netstandard/props.xml b/csharp/src/Microsoft.ML.OnnxRuntime/targets/netstandard/props.xml index 47c0202090..9376d98c04 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/targets/netstandard/props.xml +++ b/csharp/src/Microsoft.ML.OnnxRuntime/targets/netstandard/props.xml @@ -16,6 +16,12 @@ + + + $(MSBuildThisFileDirectory)../../runtimes/win-arm/native/onnxruntime.lib;%(AdditionalDependencies) + + + $(MSBuildThisFileDirectory)../../runtimes/win-x64/native/onnxruntime.lib;%(AdditionalDependencies) @@ -31,6 +37,7 @@ x86 arm64 + arm $(Platform) @@ -46,12 +53,7 @@ - - onnxruntime.dll - PreserveNewest - false - + onnxruntime.dll @@ -59,31 +61,36 @@ false + Condition="('$(PlatformTarget)' == 'x64' OR ('$(PlatformTarget)' == 'AnyCPU' AND '$(Prefer32Bit)' != 'true')) AND + Exists('$(MSBuildThisFileDirectory)..\..\runtimes\win-x64\native\onnxruntime_providers_shared.dll')"> onnxruntime_providers_shared.dll PreserveNewest false + Condition="('$(PlatformTarget)' == 'x64' OR ('$(PlatformTarget)' == 'AnyCPU' AND '$(Prefer32Bit)' != 'true')) AND + Exists('$(MSBuildThisFileDirectory)..\..\runtimes\win-x64\native\onnxruntime_providers_cuda.dll')"> onnxruntime_providers_cuda.dll PreserveNewest false + Condition="('$(PlatformTarget)' == 'x64' OR ('$(PlatformTarget)' == 'AnyCPU' AND '$(Prefer32Bit)' != 'true')) AND + Exists('$(MSBuildThisFileDirectory)..\..\runtimes\win-x64\native\onnxruntime_providers_dnnl.dll')"> onnxruntime_providers_dnnl.dll PreserveNewest false + Condition="('$(PlatformTarget)' == 'x64' OR ('$(PlatformTarget)' == 'AnyCPU' AND '$(Prefer32Bit)' != 'true')) AND + Exists('$(MSBuildThisFileDirectory)..\..\runtimes\win-x64\native\onnxruntime_providers_tensorrt.dll')"> onnxruntime_providers_tensorrt.dll PreserveNewest false + Condition="('$(PlatformTarget)' == 'x64' OR ('$(PlatformTarget)' == 'AnyCPU' AND '$(Prefer32Bit)' != 'true')) AND + Exists('$(MSBuildThisFileDirectory)..\..\runtimes\win-x64\native\onnxruntime_providers_openvino.dll')"> onnxruntime_providers_openvino.dll PreserveNewest false @@ -109,6 +116,38 @@ PreserveNewest false + + + + onnxruntime.dll + PreserveNewest + false + + + onnxruntime_providers_shared.dll + PreserveNewest + false + + + + + onnxruntime.dll + PreserveNewest + false + + + onnxruntime_providers_shared.dll + PreserveNewest + false + + + onnxruntime.dll diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj index f21509aaef..f73a01625f 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj @@ -37,6 +37,8 @@ + + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj index 853b3818ee..7ecf6b871c 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/Microsoft.ML.OnnxRuntime.Tests.Common.csproj @@ -50,6 +50,7 @@ + @@ -78,6 +79,7 @@ + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs new file mode 100644 index 0000000000..f5bf62b4b8 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/TrainingTest.cs @@ -0,0 +1,308 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.ML.OnnxRuntime.Tensors; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +// This runs in a separate package built from EndToEndTests +// and for this reason it can not refer to non-public members +// of Onnxruntime package +namespace Microsoft.ML.OnnxRuntime.Tests +{ + public partial class TrainingTest + { + private readonly ITestOutputHelper output; + + public TrainingTest(ITestOutputHelper o) + { + this.output = o; + } + + [Fact(DisplayName = "TestLoadCheckpoint")] + public void TestLoadCheckpoint() + { + string path = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var opt = new CheckpointState(path)) + { + Assert.NotNull(opt); + } + } + + [Fact(DisplayName = "TestCreateTrainingSession")] + public void TestCreateTrainingSession() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + + var trainingSession = new TrainingSession(state, trainingPath); + cleanUp.Add(trainingSession); + } + } + + [Fact(DisplayName = "TestTrainingSessionTrainStep")] + public void TestTrainingSessionTrainStep() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + + var trainingSession = new TrainingSession(state, trainingPath); + cleanUp.Add(trainingSession); + + float[] expectedOutput = TestDataLoader.LoadTensorFromFile("loss_1.out"); + var expectedOutputDimensions = new int[] { 1 }; + float[] input = TestDataLoader.LoadTensorFromFile("input-0.in"); + Int32[] labels = { 1, 1 }; + + // Run inference with pinned inputs and pinned outputs + using (DisposableListTest pinnedInputs = new DisposableListTest(), + pinnedOutputs = new DisposableListTest()) + { + var memInfo = OrtMemoryInfo.DefaultInstance; // CPU + + // Create inputs + long[] inputShape = { 2, 784 }; + pinnedInputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, input, + TensorElementType.Float, inputShape, input.Length * sizeof(float))); + + long[] labelsShape = { 2 }; + pinnedInputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, labels, + TensorElementType.Int32, labelsShape, labels.Length * sizeof(Int32))); + + + // Prepare output buffer + long[] outputShape = { }; + float[] outputBuffer = new float[expectedOutput.Length]; + pinnedOutputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, outputBuffer, + TensorElementType.Float, outputShape, outputBuffer.Length * sizeof(float))); + + trainingSession.TrainStep(pinnedInputs, pinnedOutputs); + Assert.Equal(expectedOutput, outputBuffer, new FloatComparer()); + } + } + } + + void RunTrainStep(TrainingSession trainingSession) + { + float[] expectedOutput = TestDataLoader.LoadTensorFromFile("loss_1.out"); + var expectedOutputDimensions = new int[] { 1 }; + float[] input = TestDataLoader.LoadTensorFromFile("input-0.in"); + Int32[] labels = { 1, 1 }; + + // Run inference with pinned inputs and pinned outputs + using (DisposableListTest pinnedInputs = new DisposableListTest()) + { + var memInfo = OrtMemoryInfo.DefaultInstance; // CPU + + // Create inputs + long[] inputShape = { 2, 784 }; + pinnedInputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, input, + TensorElementType.Float, inputShape, input.Length * sizeof(float))); + + long[] labelsShape = { 2 }; + pinnedInputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, labels, + TensorElementType.Int32, labelsShape, labels.Length * sizeof(Int32))); + + var outputs = trainingSession.TrainStep(pinnedInputs); + trainingSession.ResetGrad(); + outputs = trainingSession.TrainStep(pinnedInputs); + var outputBuffer = outputs.ElementAtOrDefault(0); + + Assert.Equal("542.loss", outputBuffer.Name); + Assert.Equal(OnnxValueType.ONNX_TYPE_TENSOR, outputBuffer.ValueType); + Assert.Equal(TensorElementType.Float, outputBuffer.ElementType); + + var outLabelTensor = outputBuffer.AsTensor(); + Assert.NotNull(outLabelTensor); + Assert.Equal(expectedOutput, outLabelTensor, new FloatComparer()); + } + } + + [Fact(DisplayName = "TestTrainingSessionTrainStepOrtOutput")] + public void TestTrainingSessionTrainStepOrtOutput() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + + var trainingSession = new TrainingSession(state, trainingPath); + cleanUp.Add(trainingSession); + RunTrainStep(trainingSession); + } + } + + + [Fact(DisplayName = "TestSaveCheckpoint")] + public void TestSaveCheckpoint() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + var trainingSession = new TrainingSession(state, trainingPath); + cleanUp.Add(trainingSession); + + // Save checkpoint + string savedCheckpointPath = Path.Combine(Directory.GetCurrentDirectory(), "saved_checkpoint.ckpt"); + trainingSession.SaveCheckpoint(savedCheckpointPath, false); + + // Load checkpoint and run train step + var loadedState = new CheckpointState(savedCheckpointPath); + cleanUp.Add(loadedState); + var newTrainingSession = new TrainingSession(loadedState, trainingPath); + cleanUp.Add(newTrainingSession); + RunTrainStep(newTrainingSession); + } + } + + [Fact(DisplayName = "TestTrainingSessionOptimizerStep")] + public void TestTrainingSessionOptimizerStep() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + string optimizerPath = Path.Combine(Directory.GetCurrentDirectory(), "adamw.onnx"); + + var trainingSession = new TrainingSession(state, trainingPath, optimizerPath); + cleanUp.Add(trainingSession); + + float[] expectedOutput_1 = TestDataLoader.LoadTensorFromFile("loss_1.out"); + float[] expectedOutput_2 = TestDataLoader.LoadTensorFromFile("loss_2.out"); + var expectedOutputDimensions = new int[] { 1 }; + float[] input = TestDataLoader.LoadTensorFromFile("input-0.in"); + Int32[] labels = { 1, 1 }; + + // Run train step with pinned inputs and pinned outputs + using (DisposableListTest pinnedInputs = new DisposableListTest(), + pinnedOutputs = new DisposableListTest()) + { + var memInfo = OrtMemoryInfo.DefaultInstance; // CPU + + // Create inputs + long[] inputShape = { 2, 784 }; + pinnedInputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, input, + TensorElementType.Float, inputShape, input.Length * sizeof(float))); + + long[] labelsShape = { 2 }; + pinnedInputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, labels, + TensorElementType.Int32, labelsShape, labels.Length * sizeof(Int32))); + + + // Prepare output buffer + long[] outputShape = { }; + float[] outputBuffer = new float[expectedOutput_1.Length]; + pinnedOutputs.Add(FixedBufferOnnxValue.CreateFromMemory(memInfo, outputBuffer, + TensorElementType.Float, outputShape, outputBuffer.Length * sizeof(float))); + + trainingSession.TrainStep(pinnedInputs, pinnedOutputs); + Assert.Equal(expectedOutput_1, outputBuffer, new FloatComparer()); + + trainingSession.ResetGrad(); + + trainingSession.TrainStep(pinnedInputs, pinnedOutputs); + Assert.Equal(expectedOutput_1, outputBuffer, new FloatComparer()); + + trainingSession.OptimizerStep(); + + trainingSession.TrainStep(pinnedInputs, pinnedOutputs); + Assert.Equal(expectedOutput_2, outputBuffer, new FloatComparer()); + } + } + } + + [Fact(DisplayName = "TestTrainingSessionSetLearningRate")] + public void TestTrainingSessionSetLearningRate() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + string optimizerPath = Path.Combine(Directory.GetCurrentDirectory(), "adamw.onnx"); + + var trainingSession = new TrainingSession(state, trainingPath, optimizerPath); + cleanUp.Add(trainingSession); + + float learningRate = 0.245f; + trainingSession.SetLearningRate(learningRate); + var actualLearningRate = trainingSession.GetLearningRate(); + Assert.Equal(learningRate, actualLearningRate); + } + } + + [Fact(DisplayName = "TestTrainingSessionLinearLRScheduler")] + public void TestTrainingSessionLinearLRScheduler() + { + string checkpointPath = Path.Combine(Directory.GetCurrentDirectory(), "checkpoint.ckpt"); + using (var cleanUp = new DisposableListTest()) + { + var state = new CheckpointState(checkpointPath); + cleanUp.Add(state); + Assert.NotNull(state); + string trainingPath = Path.Combine(Directory.GetCurrentDirectory(), "training_model.onnx"); + string optimizerPath = Path.Combine(Directory.GetCurrentDirectory(), "adamw.onnx"); + + var trainingSession = new TrainingSession(state, trainingPath, optimizerPath); + cleanUp.Add(trainingSession); + + float learningRate = 0.1f; + trainingSession.RegisterLinearLRScheduler(2, 4, learningRate); + RunTrainStep(trainingSession); + trainingSession.OptimizerStep(); + trainingSession.SchedulerStep(); + Assert.Equal(0.05f, trainingSession.GetLearningRate()); + trainingSession.OptimizerStep(); + trainingSession.SchedulerStep(); + Assert.Equal(0.1f, trainingSession.GetLearningRate()); + trainingSession.OptimizerStep(); + trainingSession.SchedulerStep(); + Assert.Equal(0.05f, trainingSession.GetLearningRate()); + trainingSession.OptimizerStep(); + trainingSession.SchedulerStep(); + Assert.Equal(0.0f, trainingSession.GetLearningRate()); + } + } + + internal class FloatComparer : IEqualityComparer + { + private float atol = 1e-3f; + private float rtol = 1.7e-2f; + + public bool Equals(float x, float y) + { + return Math.Abs(x - y) <= (atol + rtol * Math.Abs(y)); + } + public int GetHashCode(float x) + { + return x.GetHashCode(); + } + } + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Resources/Resource.designer.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Resources/Resource.designer.cs index e9d603525f..ce418f0916 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Resources/Resource.designer.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Droid/Resources/Resource.designer.cs @@ -13,7 +13,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests.Droid { - + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Xamarin.Android.Build.Tasks", "1.0.0.0")] public partial class Resource { diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index 393b6dd1d1..f538daf8dc 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -229,6 +229,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests { "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" }, + { "cntk_simple_seg", "Bad onnx test output caused by wrong SAME_UPPER/SAME_LOWER for ConvTranspose" }, { "coreml_Imputer-LogisticRegression_sklearn_load_breast_cancer", "Can't determine model file name" }, { "mask_rcnn_keras", "Model should be edited to remove the extra outputs" }, { "test_strnormalizer_export_monday_casesensintive_lower", "ElementType not currently supported"}, @@ -304,7 +305,6 @@ namespace Microsoft.ML.OnnxRuntime.Tests { "test_min_uint16", "node test error"}, { "test_adam_multiple", "node test error"}, { "test_loop13_seq", "node test error"}, - { "test_convtranspose_autopad_same", "node test error"}, { "test_training_dropout_default_mask", "node test error"}, { "test_min_int8", "node test error"}, { "test_identity_sequence", "data type not supported"}, diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj index 49ec3af9e9..2ae06a4efb 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj @@ -6,11 +6,11 @@ $(ProjectDir)..\.. AnyCPU;x86 bin\$(Configuration)\ - true - true - true + true + true + true - + 7.2 True true @@ -18,6 +18,9 @@ Debug;Release;RelWithDebInfo __NET_CORE_APP__;$(DefineConstants) + + + false @@ -94,6 +97,16 @@ + + + TrainingTest.cs + + + PreserveNewest + false + + + diff --git a/dockerfiles/Dockerfile.migraphx b/dockerfiles/Dockerfile.migraphx index 60d8eee6cf..19dac911a7 100644 --- a/dockerfiles/Dockerfile.migraphx +++ b/dockerfiles/Dockerfile.migraphx @@ -8,7 +8,7 @@ FROM ubuntu:18.04 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main ENV DEBIAN_FRONTEND noninteractive ENV MIGRAPHX_DISABLE_FAST_GELU=1 @@ -28,7 +28,7 @@ RUN apt-get update &&\ rocblas half aria2 libnuma-dev RUN aria2c -q -d /tmp -o cmake-3.21.0-linux-x86_64.tar.gz \ -https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz &&\ +https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz &&\ tar -zxf /tmp/cmake-3.21.0-linux-x86_64.tar.gz --strip=1 -C /usr # Install rbuild @@ -56,4 +56,3 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR /bin/sh ./build.sh --cmake_extra_defines ONNXRUNTIME_VERSION=`cat ./VERSION_NUMBER` --config Release --parallel \ --skip_tests --build_wheel --use_rocm --rocm_version=4.5.2 --rocm_home /opt/rocm --use_migraphx &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl - diff --git a/dockerfiles/Dockerfile.nuphar b/dockerfiles/Dockerfile.nuphar index 2677f08267..8f2de96246 100644 --- a/dockerfiles/Dockerfile.nuphar +++ b/dockerfiles/Dockerfile.nuphar @@ -7,7 +7,7 @@ FROM ubuntu:18.04 ARG PYTHON_VERSION=3.6 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime -ARG ONNXRUNTIME_SERVER_BRANCH=master +ARG ONNXRUNTIME_SERVER_BRANCH=main ENV DEBIAN_FRONTEND noninteractive diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino index a3647747e5..524cfeb86c 100644 --- a/dockerfiles/Dockerfile.openvino +++ b/dockerfiles/Dockerfile.openvino @@ -13,21 +13,21 @@ ENV WORKDIR_PATH=/home/openvino WORKDIR $WORKDIR_PATH ENV DEBIAN_FRONTEND noninteractive -ARG DEVICE=CPU_FP32 +ARG DEVICE=CPU_FP32 ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime.git -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/runtime/cmake USER root RUN apt update; apt install -y git protobuf-compiler libprotobuf-dev -RUN git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} +RUN git clone --recursive -b ${ONNXRUNTIME_BRANCH} ${ONNXRUNTIME_REPO} RUN /bin/sh onnxruntime/dockerfiles/scripts/install_common_deps.sh RUN ln -s cmake-* cmake-dir RUN python3 -m pip install wheel ENV PATH=${WORKDIR_PATH}/cmake-dir/bin:$PATH RUN pip3 install onnx -RUN cd onnxruntime && ./build.sh --config Release --update --build --parallel --use_openvino ${DEVICE} --build_shared_lib --build_wheel +RUN cd onnxruntime && ./build.sh --config Release --update --build --parallel --use_openvino ${DEVICE} --build_shared_lib --build_wheel #Steps to download sources RUN cat /etc/apt/sources.list | sed 's/^# deb-src/deb-src/g' > ./temp; mv temp /etc/apt/sources.list diff --git a/dockerfiles/Dockerfile.openvino-centos7 b/dockerfiles/Dockerfile.openvino-centos7 index e500f3ce5a..4644187e53 100755 --- a/dockerfiles/Dockerfile.openvino-centos7 +++ b/dockerfiles/Dockerfile.openvino-centos7 @@ -11,7 +11,7 @@ ARG MY_ROOT=/code ARG YUM_OV_PACKAGE=intel-openvino-runtime-centos7-2021.4.752.x86_64 ARG DEVICE=CPU_FP32 ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.4.752 ENV InferenceEngine_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/share @@ -24,7 +24,7 @@ ENV HDDL_INSTALL_DIR=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/ext ENV LD_LIBRARY_PATH=${INTEL_OPENVINO_DIR}/deployment_tools/inference_engine/external/hddl/lib:$LD_LIBRARY_PATH ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/lib:/usr/local/lib64:/usr/lib64:/lib64:$LD_LIBRARY_PATH -# Install packages +# Install packages RUN yum update -y && \ yum groupinstall "Development Tools" -y && \ yum install -y yum-utils autoconf automake libtool unzip udev wget zlib-devel libffi-devel openssl-devel boost-devel-1.53.0 && \ @@ -42,7 +42,7 @@ RUN yum update -y && \ cd /opt/ && wget https://github.com/libusb/libusb/archive/v1.0.22.zip && \ unzip v1.0.22.zip && rm -rf v1.0.22.zip && cd /opt/libusb-1.0.22 && \ # bootstrap steps - ./bootstrap.sh && \ + ./bootstrap.sh && \ ./configure --disable-udev --enable-shared && \ make -j4 && \ # configure libusb1.0.22 @@ -64,7 +64,7 @@ RUN yum update -y && \ cp /opt/intel/openvino_2021/deployment_tools/inference_engine/external/97-myriad-usbboot.rules /etc/udev/rules.d/ && \ ldconfig && \ # Install GPU runtime and drivers - cd ${MY_ROOT} && \ + cd ${MY_ROOT} && \ mkdir /tmp/opencl && \ cd /tmp/opencl && \ yum install -y epel-release && \ @@ -87,8 +87,8 @@ RUN yum update -y && \ cd /code/ && \ wget https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz && tar xvf Python-3.8.3.tgz && \ cd Python-3.8*/ && ./configure && make && make install && \ - cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.3/ /usr/bin/Python38 && \ -# installing dependancies + cd ../ && mkdir -p /usr/bin/Python38 && ln -s Python-3.8.3/ /usr/bin/Python38 && \ +# installing dependancies yum install -y python3-lxml python3-six libusb.x86_64 && \ yum clean packages && yum clean all && rm -rf /var/cache/yum && \ # Build onnxruntime @@ -102,4 +102,4 @@ RUN yum update -y && \ cd $MY_ROOT && rm -rf onnxruntime Python-3* && \ cd ${MY_ROOT}/ && rm -rf cmake* && \ cd /usr/share/ && rm -rf gcc* && cd /usr/lib/ && rm -rf gcc cd && rm -rf .cache && \ - cd ${INTEL_OPENVINO_DIR}/ && rm -rf documentation data_processing && cd deployment_tools/ && rm -rf tools \ No newline at end of file + cd ${INTEL_OPENVINO_DIR}/ && rm -rf documentation data_processing && cd deployment_tools/ && rm -rf tools diff --git a/dockerfiles/Dockerfile.openvino-csharp b/dockerfiles/Dockerfile.openvino-csharp index 1daaa7c945..6f98b156a7 100644 --- a/dockerfiles/Dockerfile.openvino-csharp +++ b/dockerfiles/Dockerfile.openvino-csharp @@ -7,7 +7,7 @@ FROM ubuntu:18.04 ARG DEVICE=CPU_FP32 ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime.git -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main WORKDIR /code ARG MY_ROOT=/code diff --git a/dockerfiles/Dockerfile.rocm b/dockerfiles/Dockerfile.rocm index 68412b0104..19b65d3e38 100644 --- a/dockerfiles/Dockerfile.rocm +++ b/dockerfiles/Dockerfile.rocm @@ -8,7 +8,7 @@ FROM ubuntu:18.04 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main ENV DEBIAN_FRONTEND noninteractive RUN apt-get clean && apt-get update && apt-get install -y locales @@ -23,7 +23,7 @@ RUN apt-get update && apt-get install -y gnupg2 --no-install-recommends curl && sh -c 'echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/4.5.2/ ubuntu main > /etc/apt/sources.list.d/rocm.list' RUN apt-get update &&\ - apt-get install -y sudo git bash build-essential rocm-dev libpython3.6-dev python3-pip miopen-hip rocblas half aria2 libnuma-dev + apt-get install -y sudo git bash build-essential rocm-dev libpython3.6-dev python3-pip miopen-hip rocblas half aria2 libnuma-dev RUN aria2c -q -d /tmp -o cmake-3.20.0-linux-x86_64.tar.gz \ https://github.com/Kitware/CMake/releases/download/v3.20.0/cmake-3.20.0-linux-x86_64.tar.gz &&\ @@ -48,4 +48,3 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --use_rocm --rocm_home=/opt/rocm &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. - diff --git a/dockerfiles/Dockerfile.tensorrt b/dockerfiles/Dockerfile.tensorrt index 47a47fc61d..8c31b999b6 100644 --- a/dockerfiles/Dockerfile.tensorrt +++ b/dockerfiles/Dockerfile.tensorrt @@ -9,7 +9,7 @@ ARG TRT_CONTAINER_VERSION=21.12 FROM nvcr.io/nvidia/tensorrt:${TRT_CONTAINER_VERSION}-py3 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main ARG CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80 RUN apt-get update &&\ @@ -27,4 +27,4 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR cd onnxruntime &&\ /bin/sh build.sh --parallel --build_shared_lib --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ --config Release --build_wheel --skip_tests --skip_submodule_sync --cmake_extra_defines '"CMAKE_CUDA_ARCHITECTURES='${CMAKE_CUDA_ARCHITECTURES}'"' &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ - cd .. + cd .. diff --git a/dockerfiles/Dockerfile.vitisai b/dockerfiles/Dockerfile.vitisai index 7dcbaa403a..1cc73bb1e2 100644 --- a/dockerfiles/Dockerfile.vitisai +++ b/dockerfiles/Dockerfile.vitisai @@ -7,7 +7,7 @@ FROM xilinx/vitis-ai-cpu:1.3.598 ARG ONNXRUNTIME_REPO=https://github.com/Microsoft/onnxruntime -ARG ONNXRUNTIME_BRANCH=master +ARG ONNXRUNTIME_BRANCH=main ARG PYXIR_REPO=https://github.com/Xilinx/pyxir ARG PYXIR_BRANCH=v0.2.0 diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 534b4bb668..5ef8500457 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -10,7 +10,7 @@ **Platforms** - ARM 32v7: [Dockerfile](Dockerfile.arm32v7), [Instructions](#arm-3264) -- ARM 64: [Dockerfile](Dockerfile.arm64), [Instructions](#arm-3264) +- ARM 64: [Dockerfile](Dockerfile.arm64), [Instructions](#arm-3264) - NVIDIA Jetson TX1/TX2/Nano/Xavier: [Dockerfile](Dockerfile.jetson), [Instructions](#nvidia-jetson-tx1tx2nanoxavier) **Other** @@ -88,7 +88,7 @@ git submodule update --init ### **1. Using pre-built container images for Python API** -The unified container image from [Dockerhub](https://hub.docker.com/repository/docker/openvino/onnxruntime_ep_ubuntu18) can be used to run an application on any of the target accelerators. In order to select the target accelerator, the application should explicitly specifiy the choice using the `device_type` configuration option for OpenVINO Execution provider. Refer to [OpenVINO EP runtime configuration documentation](https://www.onnxruntime.ai/docs/reference/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options) for details on specifying this option in the application code. +The unified container image from [Dockerhub](https://hub.docker.com/repository/docker/openvino/onnxruntime_ep_ubuntu18) can be used to run an application on any of the target accelerators. In order to select the target accelerator, the application should explicitly specifiy the choice using the `device_type` configuration option for OpenVINO Execution provider. Refer to [OpenVINO EP runtime configuration documentation](https://www.onnxruntime.ai/docs/reference/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options) for details on specifying this option in the application code. If the `device_type` runtime config option is not explicitly specified, CPU will be chosen as the hardware target execution. ### **2. Building from Dockerfile** @@ -96,7 +96,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will Retrieve your docker image in one of the following ways. - - Choose Dockerfile.openvino for Python API or Dockerfile.openvino-csharp for C# API as for building an OpenVINO 2021.3 based Docker image. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is master. + - Choose Dockerfile.openvino for Python API or Dockerfile.openvino-csharp for C# API as for building an OpenVINO 2021.3 based Docker image. Providing the docker build argument DEVICE enables the onnxruntime build for that particular device. You can also provide arguments ONNXRUNTIME_REPO and ONNXRUNTIME_BRANCH to test that particular repo and branch. Default repository is http://github.com/microsoft/onnxruntime and default branch is main. ``` docker build --rm -t onnxruntime --build-arg DEVICE=$DEVICE -f . ``` @@ -127,7 +127,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will Example: HETERO:MYRIAD,CPU HETERO:HDDL,GPU,CPU MULTI:MYRIAD,GPU,CPU AUTO:GPU,CPU -*This is the hardware accelerator target that is enabled by **default** in the container image. After building the container image for one default target, the application may explicitly choose a different target at run time with the same container by using the [Dynamic device selction API](https://github.com/microsoft/onnxruntime/blob/master/docs/execution_providers/OpenVINO-ExecutionProvider.md#dynamic-device-selection).* +*This is the hardware accelerator target that is enabled by **default** in the container image. After building the container image for one default target, the application may explicitly choose a different target at run time with the same container by using the [Dynamic device selction API](https://github.com/microsoft/onnxruntime/blob/main/docs/execution_providers/OpenVINO-ExecutionProvider.md#dynamic-device-selection).* ### OpenVINO on CPU @@ -223,7 +223,7 @@ If the `device_type` runtime config option is not explicitly specified, CPU will ## ARM 32/64 The build instructions are similar to x86 CPU. But if you want to build them on a x86 machine, you need to install qemu-user-static system package (outside of docker instances) first. Then - + 1. Update submodules ``` git submodule update --init @@ -241,7 +241,7 @@ git submodule update --init ``` For ARM32, please use Dockerfile.arm32v7 instead of Dockerfile.arm64. - + ## NVIDIA Jetson TX1/TX2/Nano/Xavier: These instructions are for [JetPack SDK 4.4](https://developer.nvidia.com/embedded/jetpack). @@ -299,7 +299,7 @@ git submodule update --init docker run -it onnxruntime-nuphar ``` -## MIGraphX +## MIGraphX **Ubuntu 18.04, rocm4.5, AMDMIGraphX v1.2** 1. Build the docker image from the Dockerfile in this repository. diff --git a/docs/C_API_Guidelines.md b/docs/C_API_Guidelines.md index e00ac7b96c..3416d0a504 100644 --- a/docs/C_API_Guidelines.md +++ b/docs/C_API_Guidelines.md @@ -36,13 +36,13 @@ If an API such as CreateSession creates an Ort object such as Session, Session c No C++ exceptions must propagate through the C++/C boundaries. All C++ exceptions must be converted to OrtStatus instances at API boundaries. Such functions should return nullptr on success. -Macros API_IMPL_BEGIN and API_IMPL_END are helpful in this regard. +Macros API_IMPL_BEGIN and API_IMPL_END are helpful in this regard. Cleanup API that destroys objects or simply deallocates memory must return void. Most of the time such API can never error out. Adding return status creates more uncertainty for the client and does not help in exception scenarios such as try/finally in C#. Returning void helps clients to write cleaner code and preserve original exception if any with its meaningful error message rather than memory deallocation failure. This requirement will also help us to create C++ API wrappers that are exception safe. -Consider logging errors if you must rather than return them to the client. +Consider logging errors if you must rather than return them to the client. Example: on Windows delete operator is implemented on top of HeapFree() which may return an error. However, delete never returns anything and can be relied upon as a no throw primitive for cleanup purposes. @@ -52,7 +52,7 @@ When API errors out it must leave all its out parameters and buffers untouched, The obvious exception in this rule is the actual OrtStatus that is dynamically allocated and must be released by the client using the corresponding API. -Some of the client code, notably in C#, attempts to detect which out arguments need a cleanup when an API errors out. The way it is done, out arguments are pre-set to a specific value, such as zero. If the API errors out, the client code attempts to cleanup if the out argument has changed. +Some of the client code, notably in C#, attempts to detect which out arguments need a cleanup when an API errors out. The way it is done, out arguments are pre-set to a specific value, such as zero. If the API errors out, the client code attempts to cleanup if the out argument has changed. Such a technique is error prone and dangerous, as the client has no way of finding out if the out argument has already been cleaned up by the API as should be the case. It may result in double free. One reason for this is our insufficient documentation. This also results in a convoluted hard to read code with nested try/finally/catch clauses. @@ -80,4 +80,4 @@ Use types that fall into established patterns. For example, we use int64_t for d ### 9. Adding a new API -Follow these guidelines and instructions in the source code. "Rules on how to add a new Ort API version" in [onnxruntime_c_api.cc](https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/core/session/onnxruntime_c_api.cc). +Follow these guidelines and instructions in the source code. "Rules on how to add a new Ort API version" in [onnxruntime_c_api.cc](https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/core/session/onnxruntime_c_api.cc). diff --git a/docs/Coding_Conventions_and_Standards.md b/docs/Coding_Conventions_and_Standards.md index e1e0ec08bd..d08875ddbb 100644 --- a/docs/Coding_Conventions_and_Standards.md +++ b/docs/Coding_Conventions_and_Standards.md @@ -97,7 +97,7 @@ void foo(gsl::span names) { * Qualify usages of `auto` with `const`, `*`, `&` and `&&` where applicable to more clearly express the intent * When adding a new class, disable copy/assignment/move until you have a proven need for these capabilities. If a need arises, enable copy/assignment/move selectively, and when doing so validate that the implementation of the class supports what is being enabled. * Use `ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE` initially - * See the other `ORT_DISALLOW_*` macros in + * See the other `ORT_DISALLOW_*` macros in * Sometimes, `std::unique_ptr` might be considered for delayed or optional construction of objects or members of classes. Instead, use `std::optional` as appropriate to reduce the number of allocations. * Don't use `else` after `return`. see: [https://llvm.org/docs/CodingStandards.html#don-t-use-else-after-a-return](https://llvm.org/docs/CodingStandards.html#don-t-use-else-after-a-return) * Don't overuse `std::shared_ptr`. Use `std::shared_ptr` only if it's not clear when and where the object will be de-allocated. See also: [https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-shared_ptr](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rf-shared_ptr) @@ -141,7 +141,7 @@ Follow the [Black formatter](https://black.readthedocs.io)'s coding style when p Please adhere to the [PEP8 Style Guide](https://www.python.org/dev/peps/pep-0008/). We use [Google's python style guide](https://google.github.io/styleguide/pyguide.html) as the style guide which is an extension to PEP8. -Code can be validated with [flake8](https://pypi.org/project/flake8/) using the configuration file in the root directory called [.flake8](https://github.com/microsoft/onnxruntime/tree/master/.flake8). +Code can be validated with [flake8](https://pypi.org/project/flake8/) using the configuration file in the root directory called [.flake8](https://github.com/microsoft/onnxruntime/blob/main/.flake8). Use `pyright`, which is provided as a component of the `pylance` extension in VS Code for static type checking. diff --git a/docs/ContribOperators.md b/docs/ContribOperators.md index 2dca2eca8b..e1a8d23a0a 100644 --- a/docs/ContribOperators.md +++ b/docs/ContribOperators.md @@ -1,5 +1,5 @@ ## Contrib Operator Schemas -*This file is automatically generated from the registered contrib operator schemas by [this script](https://github.com/microsoft/onnxruntime/blob/master/tools/python/gen_contrib_doc.py). +*This file is automatically generated from the registered contrib operator schemas by [this script](https://github.com/microsoft/onnxruntime/blob/main/tools/python/gen_contrib_doc.py). Do not modify directly.* * com.microsoft @@ -56,6 +56,9 @@ Do not modify directly.* * com.microsoft.QLinearReduceMean * com.microsoft.QLinearSigmoid * com.microsoft.QLinearSoftmax + * com.microsoft.QOrderedGelu + * com.microsoft.QOrderedLayerNormalization + * com.microsoft.QOrderedMatMul * com.microsoft.QuantizeLinear * com.microsoft.Range * com.microsoft.ReduceSumInteger @@ -1357,14 +1360,14 @@ This version of the operator has been available since version 1 of the 'com.micr
      Whether B should be transposed
      -#### Inputs +#### Inputs (2 - 3)
      A : T
      Input tensor A. The shape of A should be (M, K) if transA is 0, or (K, M) if transA is non-zero.
      B : T
      Input tensor B. The shape of B should be (K, N) if transB is 0, or (N, K) if transB is non-zero.
      -
      C : T
      +
      C (optional) : T
      Input tensor C. The shape of C should be unidirectional broadcastable to (M, N).
      @@ -2862,6 +2865,163 @@ This version of the operator has been available since version 1 of the 'com.micr +### **com.microsoft.QOrderedGelu** + + Ordered Quantize Gelu. + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
      +
      order_X : int
      +
      cublasLt order of input X. Default is ROW MAJOR.
      +
      order_Y : int
      +
      cublasLt order of matrix Y, must be same as order_X. Default is ROW MAJOR.
      +
      + +#### Inputs + +
      +
      X : Q
      +
      N-dimensional input A
      +
      scale_X : S
      +
      scale of the input A
      +
      scale_Y : S
      +
      scale of the output Y
      +
      + +#### Outputs + +
      +
      Y : Q
      +
      Output of the Gelu
      +
      + +#### Type Constraints + +
      +
      Q : tensor(int8)
      +
      Constrain input and output types to int8 tensors.
      +
      S : tensor(float)
      +
      Constrain scales to float32
      +
      + + +### **com.microsoft.QOrderedLayerNormalization** + + QOrderedLayerNormalization + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
      +
      axis : int
      +
      The first normalization dimension: normalization will be performed along dimensions axis : rank(inputs).
      +
      epsilon : float
      +
      The epsilon value to use to avoid division by zero.
      +
      order_X : int
      +
      cublasLt order of input X. Default is ROW MAJOR.
      +
      order_Y : int
      +
      cublasLt order of matrix Y, must be same as order_X. Default is ROW MAJOR.
      +
      + +#### Inputs + +
      +
      X : Q
      +
      Input data tensor from the previous layer.
      +
      scale_X : S
      +
      scale of the quantized X
      +
      scale : F
      +
      Scale tensor, i.e., gamma vector.
      +
      B (optional) : F
      +
      Bias tensor.
      +
      scale_Y : S
      +
      scale of the quantized X
      +
      + +#### Outputs + +
      +
      Y : Q
      +
      Output data tensor.
      +
      + +#### Type Constraints + +
      +
      F : tensor(float16), tensor(float)
      +
      Constrain input gamma and bias could be float16/float tensors. float may get better precision, float16 runs faster.
      +
      S : tensor(float)
      +
      quantization scale must be float tensors.
      +
      Q : tensor(int8)
      +
      quantization tensor must be int8 tensors.
      +
      + + +### **com.microsoft.QOrderedMatMul** + + TODO + +#### Version + +This version of the operator has been available since version 1 of the 'com.microsoft' operator set. + +#### Attributes + +
      +
      order_A : int
      +
      cublasLt order of matrix A. Default is ROW MAJOR.
      +
      order_B : int
      +
      cublasLt order of matrix B. Default is ROW MAJOR.
      +
      order_Y : int
      +
      cublasLt order of matrix Y and optional matrix C. Default is ROW MAJOR.
      +
      + +#### Inputs (5 - 8) + +
      +
      A : Q
      +
      3-dimensional matrix A
      +
      scale_A : S
      +
      scale of the input A
      +
      B : Q
      +
      2-dimensional matrix B
      +
      scale_B : S
      +
      scale of the input B
      +
      scale_Y : S
      +
      scale of the output Y
      +
      bias (optional) : S
      +
      1d bias
      +
      C (optional) : Q
      +
      3d or 2d matrix C. if 2d expand to 3d first. Shape[0] should be 1 or same as A.shape[0]
      +
      scale_C (optional) : S
      +
      scale of the input A
      +
      + +#### Outputs + +
      +
      Y : Q
      +
      Matrix multiply results from A * B
      +
      + +#### Type Constraints + +
      +
      Q : tensor(int8)
      +
      Constrain input and output types to int8 tensors.
      +
      S : tensor(float)
      +
      Constrain bias and scales to float32
      +
      + + ### **com.microsoft.QuantizeLinear** The linear quantization operator. It consumes a full precision data, a scale, a zero point to compute the low precision / quantized tensor. diff --git a/docs/NotesOnThreading.md b/docs/NotesOnThreading.md index 25f504c02c..83bb0d400a 100644 --- a/docs/NotesOnThreading.md +++ b/docs/NotesOnThreading.md @@ -3,12 +3,12 @@ This document is intended for ORT developers. ORT allows the usage of either OpenMP or non-OpenMP (ORT) threads for execution. Threadpool management -is abstracted behind: (1) ThreadPool class in [threadpool.h](https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/platform/threadpool.h) and (2) functions in [thread_utils.h](https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/core/util/thread_utils.h). +is abstracted behind: (1) ThreadPool class in [threadpool.h](https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/platform/threadpool.h) and (2) functions in [thread_utils.h](https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/core/util/thread_utils.h). When developing an op, please use these abstractions to parallelize your code. These abstractions centralize 2 things. When OpenMP is enabled, they resort to using OpenMP. When OpenMP is disabled they resort to sequential execution if the threadpool ptr is NULL or schedule the tasks on the threadpool otherwise. -Examples of these abstractions are: ([threadpool.h](https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/platform/threadpool.h) has more documentation for these) +Examples of these abstractions are: ([threadpool.h](https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/platform/threadpool.h) has more documentation for these) * TryParallelFor * TrySimpleParallelFor * TryBatchParallelFor diff --git a/docs/ORTMobilePackageOperatorTypeSupport.md b/docs/ORTMobilePackageOperatorTypeSupport.md index 09de5d9d4c..6a69a2c598 100644 --- a/docs/ORTMobilePackageOperatorTypeSupport.md +++ b/docs/ORTMobilePackageOperatorTypeSupport.md @@ -2,7 +2,7 @@ ## Supported operators and types -The supported operators and types are based on what is required to support float32 and quantized versions of popular models. The full list of input models used to determine this list is available [here](https://github.com/microsoft/onnxruntime/blob/master/tools/ci_build/github/android/mobile_package.required_operators.readme.txt) +The supported operators and types are based on what is required to support float32 and quantized versions of popular models. The full list of input models used to determine this list is available [here](https://github.com/microsoft/onnxruntime/blob/main/tools/ci_build/github/android/mobile_package.required_operators.readme.txt) ## Supported data input types diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index f50b920234..680979ca8a 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -1,11 +1,12 @@ ## Supported Operators and Data Types -*This file is automatically generated from the registered kernels by [this script](https://github.com/microsoft/onnxruntime/blob/master/tools/python/gen_opkernel_doc.py). +*This file is automatically generated from the registered kernels by [this script](https://github.com/microsoft/onnxruntime/blob/main/tools/python/gen_opkernel_doc.py). Do not modify directly.* ## Execution Providers - [CPUExecutionProvider](#cpuexecutionprovider) - [CUDAExecutionProvider](#cudaexecutionprovider) +- [DmlExecutionProvider](#dmlexecutionprovider) --------------- @@ -755,6 +756,311 @@ Do not modify directly.* |Xor|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|7+|**T** = tensor(bool)
      **T1** = tensor(bool)| | | | | + + +## Operators implemented by DmlExecutionProvider + +| Op Name | Parameters | OpSet Version | Types Supported | +|---------|------------|---------------|-----------------| +|**Operator Domain:** *ai.onnx*|||| +|Abs|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||6+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|Acos|*in* input:**T**
      *out* output:**T**|7+|**T** = tensor(float), tensor(float16)| +|Acosh|*in* input:**T**
      *out* output:**T**|9+|**T** = tensor(float), tensor(float16)| +|Add|*in* A:**T**
      *in* B:**T**
      *out* C:**T**|14+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||7+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Affine|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| +|And|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|7+|**T** = tensor(bool)| +|ArgMax|*in* data:**T**
      *out* reduced:**tensor(int64)**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ArgMin|*in* data:**T**
      *out* reduced:**tensor(int64)**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Asin|*in* input:**T**
      *out* output:**T**|7+|**T** = tensor(float), tensor(float16)| +|Asinh|*in* input:**T**
      *out* output:**T**|9+|**T** = tensor(float), tensor(float16)| +|Atan|*in* input:**T**
      *out* output:**T**|7+|**T** = tensor(float), tensor(float16)| +|Atanh|*in* input:**T**
      *out* output:**T**|9+|**T** = tensor(float), tensor(float16)| +|AveragePool|*in* X:**T**
      *out* Y:**T**|11+|**T** = tensor(float), tensor(float16)| +|||10+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|BatchNormalization|*in* X:**T**
      *in* scale:**T**
      *in* B:**T**
      *in* input_mean:**U**
      *in* input_var:**U**
      *out* Y:**T**
      *out* running_mean:**U**
      *out* running_var:**U**

      or

      *in* X:**T**
      *in* scale:**T**
      *in* B:**T**
      *in* mean:**T**
      *in* var:**T**
      *out* Y:**T**
      *out* mean:**T**
      *out* var:**T**
      *out* saved_mean:**T**
      *out* saved_var:**T**

      or

      *in* X:**T**
      *in* scale:**T1**
      *in* B:**T1**
      *in* input_mean:**T2**
      *in* input_var:**T2**
      *out* Y:**T**
      *out* running_mean:**T2**
      *out* running_var:**T2**|15+|**T** = tensor(float), tensor(float16)| +|||14+|**T** = tensor(float), tensor(float16)| +|||9+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|BitShift|*in* X:**T**
      *in* Y:**T**
      *out* Z:**T**|11+|**T** = tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Cast|*in* input:**T1**
      *out* output:**T2**|13+|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||9+|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||6+|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|CastLike|*in* input:**T1**
      *in* target_type:**T2**
      *out* output:**T2**|15+|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Ceil|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Celu|*in* X:**T**
      *out* Y:**T**|12+|**T** = tensor(float), tensor(float16)| +|Clip|*in* input:**T**
      *in* min:**T**
      *in* max:**T**
      *out* output:**T**

      or

      *in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Concat|*in* inputs:**T**
      *out* concat_result:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||4+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ConstantOfShape|*in* input:**T1**
      *out* output:**T2**|9+|**T1** = tensor(int64)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Conv|*in* X:**T**
      *in* W:**T**
      *in* B:**T**
      *out* Y:**T**|11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ConvInteger|*in* x:**T1**
      *in* w:**T2**
      *in* x_zero_point:**T1**
      *in* w_zero_point:**T2**
      *out* y:**T3**|10+|**T1** = tensor(int8), tensor(uint8)
      **T2** = tensor(int8), tensor(uint8)
      **T3** = tensor(int32)| +|ConvTranspose|*in* X:**T**
      *in* W:**T**
      *in* B:**T**
      *out* Y:**T**|11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|Cos|*in* input:**T**
      *out* output:**T**|7+|**T** = tensor(float), tensor(float16)| +|Cosh|*in* input:**T**
      *out* output:**T**|9+|**T** = tensor(float), tensor(float16)| +|Crop|*in* input:**T**
      *out* output:**T**|1+|**T** = tensor(float), tensor(float16)| +|CumSum|*in* x:**T**
      *in* axis:**T2**
      *out* y:**T**|14+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|DepthToSpace|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|DequantizeLinear|*in* x:**T**
      *in* x_scale:**tensor(float)**
      *in* x_zero_point:**T**
      *out* y:**tensor(float)**|13+|**T** = tensor(int32), tensor(int8), tensor(uint8)| +|||10+|**T** = tensor(int32), tensor(int8), tensor(uint8)| +|Div|*in* A:**T**
      *in* B:**T**
      *out* C:**T**|14+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||7+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|Dropout|*in* data:**T**
      *in* ratio:**T1**
      *in* training_mode:**T2**
      *out* output:**T**
      *out* mask:**T2**

      or

      *in* data:**T**
      *out* output:**T**
      *out* mask:**T**

      or

      *in* data:**T**
      *out* output:**T**
      *out* mask:**T1**|7+|**T** = tensor(float), tensor(float16)| +|DynamicQuantizeLinear|*in* x:**T1**
      *out* y:**T2**
      *out* y_scale:**tensor(float)**
      *out* y_zero_point:**T2**|11+|**T1** = tensor(float)
      **T2** = tensor(uint8)| +|Einsum|*in* Inputs:**T**
      *out* Output:**T**|12+|**T** = tensor(float), tensor(float16)| +|Elu|*in* X:**T**
      *out* Y:**T**|6+|**T** = tensor(float), tensor(float16)| +|Equal|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|||7+|**T** = tensor(float), tensor(float16)
      **T1** = tensor(bool)| +|Erf|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||9+|**T** = tensor(float), tensor(float16)| +|Exp|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Expand|*in* input:**T**
      *in* shape:**tensor(int64)**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||8+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|EyeLike|*in* input:**T1**
      *out* output:**T2**|9+|**T1** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Flatten|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||9+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Floor|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|GRU|*in* X:**T**
      *in* W:**T**
      *in* R:**T**
      *in* B:**T**
      *in* sequence_lens:**T1**
      *in* initial_h:**T**
      *out* Y:**T**
      *out* Y_h:**T**|14+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|Gather|*in* data:**T**
      *in* indices:**Tind**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|GatherElements|*in* data:**T**
      *in* indices:**Tind**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|GatherND|*in* data:**T**
      *in* indices:**tensor(int64)**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Gemm|*in* A:**T**
      *in* B:**T**
      *in* C:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||9+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|GlobalAveragePool|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| +|GlobalLpPool|*in* X:**T**
      *out* Y:**T**|2+|**T** = tensor(float), tensor(float16)| +|GlobalMaxPool|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| +|Greater|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|||9+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|||7+|**T** = tensor(float), tensor(float16)
      **T1** = tensor(bool)| +|GreaterOrEqual|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|HardSigmoid|*in* X:**T**
      *out* Y:**T**|6+|**T** = tensor(float), tensor(float16)| +|Hardmax|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|Identity|*in* input:**T**
      *out* output:**T**

      or

      *in* input:**V**
      *out* output:**V**|14+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ImageScaler|*in* input:**T**
      *out* output:**T**|1+|**T** = tensor(float), tensor(float16)| +|InstanceNormalization|*in* input:**T**
      *in* scale:**T**
      *in* B:**T**
      *out* output:**T**|6+|**T** = tensor(float), tensor(float16)| +|IsInf|*in* X:**T1**
      *out* Y:**T2**|10+|**T1** = tensor(float)
      **T2** = tensor(bool)| +|IsNaN|*in* X:**T1**
      *out* Y:**T2**|13+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(bool)| +|||9+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(bool)| +|LRN|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|LSTM|*in* X:**T**
      *in* W:**T**
      *in* R:**T**
      *in* B:**T**
      *in* sequence_lens:**T1**
      *in* initial_h:**T**
      *in* initial_c:**T**
      *in* P:**T**
      *out* Y:**T**
      *out* Y_h:**T**
      *out* Y_c:**T**|14+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|LeakyRelu|*in* X:**T**
      *out* Y:**T**|6+|**T** = tensor(float), tensor(float16)| +|Less|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|||9+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|||7+|**T** = tensor(float), tensor(float16)
      **T1** = tensor(bool)| +|LessOrEqual|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T1** = tensor(bool)| +|Log|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|LogSoftmax|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|LpNormalization|*in* input:**T**
      *out* output:**T**|1+|**T** = tensor(float), tensor(float16)| +|LpPool|*in* X:**T**
      *out* Y:**T**|11+|**T** = tensor(float), tensor(float16)| +|||2+|**T** = tensor(float), tensor(float16)| +|MatMul|*in* A:**T**
      *in* B:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||9+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|MatMulInteger|*in* A:**T1**
      *in* B:**T2**
      *in* a_zero_point:**T1**
      *in* b_zero_point:**T2**
      *out* Y:**T3**|10+|**T1** = tensor(int8), tensor(uint8)
      **T2** = tensor(int8), tensor(uint8)
      **T3** = tensor(int32)| +|Max|*in* data_0:**T**
      *out* max:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||8+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|MaxPool|*in* X:**T**
      *out* Y:**T**

      or

      *in* X:**T**
      *out* Y:**T**
      *out* Indices:**I**|12+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int8), tensor(uint8)| +|||11+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int8), tensor(uint8)| +|||10+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int8), tensor(uint8)| +|||8+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int8), tensor(uint8)| +|||1+|**T** = tensor(float), tensor(float16)| +|MaxRoiPool|*in* X:**T**
      *in* rois:**T**
      *out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| +|MaxUnpool|*in* X:**T1**
      *in* I:**T2**
      *in* output_shape:**T2**
      *out* output:**T1**|11+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(int64)| +|||9+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(int64)| +|Mean|*in* data_0:**T**
      *out* mean:**T**|13+|**T** = tensor(float), tensor(float16)| +|||8+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|MeanVarianceNormalization|*in* X:**T**
      *out* Y:**T**

      or

      *in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||9+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|MemcpyFromHost|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|MemcpyToHost|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Min|*in* data_0:**T**
      *out* min:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||8+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Mod|*in* A:**T**
      *in* B:**T**
      *out* C:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||10+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|Mul|*in* A:**T**
      *in* B:**T**
      *out* C:**T**|14+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||7+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Neg|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8)| +|||6+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8)| +|Not|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(bool)| +|OneHot|*in* indices:**T1**
      *in* depth:**T2**
      *in* values:**T3**
      *out* output:**T3**|11+|**T1** = tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T3** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||9+|**T1** = tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)
      **T2** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **T3** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Or|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|7+|**T** = tensor(bool)| +|PRelu|*in* X:**T**
      *in* slope:**T**
      *out* Y:**T**|9+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8)| +|||7+|**T** = tensor(float), tensor(float16)| +|Pad|*in* data:**T**
      *in* pads:**tensor(int64)**
      *in* constant_value:**T**
      *out* output:**T**

      or

      *in* data:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||2+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|ParametricSoftplus|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| +|Pow|*in* X:**T**
      *in* Y:**T**
      *out* Z:**T**

      or

      *in* X:**T**
      *in* Y:**T1**
      *out* Z:**T**|15+|**T** = tensor(float), tensor(float16), tensor(int32)
      **T1** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||13+|**T** = tensor(float), tensor(float16), tensor(int32)
      **T1** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int32)
      **T1** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||7+|**T** = tensor(float), tensor(float16)| +|QLinearConv|*in* x:**T1**
      *in* x_scale:**tensor(float)**
      *in* x_zero_point:**T1**
      *in* w:**T2**
      *in* w_scale:**tensor(float)**
      *in* w_zero_point:**T2**
      *in* y_scale:**tensor(float)**
      *in* y_zero_point:**T3**
      *in* B:**T4**
      *out* y:**T3**|10+|**T1** = tensor(int8), tensor(uint8)
      **T2** = tensor(int8), tensor(uint8)
      **T3** = tensor(int8), tensor(uint8)
      **T4** = tensor(int32)| +|QLinearMatMul|*in* a:**T1**
      *in* a_scale:**tensor(float)**
      *in* a_zero_point:**T1**
      *in* b:**T2**
      *in* b_scale:**tensor(float)**
      *in* b_zero_point:**T2**
      *in* y_scale:**tensor(float)**
      *in* y_zero_point:**T3**
      *out* y:**T3**|10+|**T1** = tensor(int8), tensor(uint8)
      **T2** = tensor(int8), tensor(uint8)
      **T3** = tensor(int8), tensor(uint8)| +|QuantizeLinear|*in* x:**T1**
      *in* y_scale:**tensor(float)**
      *in* y_zero_point:**T2**
      *out* y:**T2**|13+|**T1** = tensor(float), tensor(int32)
      **T2** = tensor(int8), tensor(uint8)| +|||10+|**T1** = tensor(float), tensor(int32)
      **T2** = tensor(int8), tensor(uint8)| +|RNN|*in* X:**T**
      *in* W:**T**
      *in* R:**T**
      *in* B:**T**
      *in* sequence_lens:**T1**
      *in* initial_h:**T**
      *out* Y:**T**
      *out* Y_h:**T**|14+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|Range|*in* start:**T**
      *in* limit:**T**
      *in* delta:**T**
      *out* output:**T**|11+|**T** = tensor(float), tensor(int16), tensor(int32), tensor(int64)| +|Reciprocal|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|ReduceL1|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||1+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|ReduceL2|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ReduceLogSum|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ReduceLogSumExp|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ReduceMax|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ReduceMean|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ReduceMin|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||12+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|ReduceProd|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||1+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|ReduceSum|*in* data:**T**
      *in* axes:**tensor(int64)**
      *out* reduced:**T**

      or

      *in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||1+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|ReduceSumSquare|*in* data:**T**
      *out* reduced:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||11+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|||1+|**T** = tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)| +|Relu|*in* X:**T**
      *out* Y:**T**|14+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8)| +|||13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Reshape|*in* data:**T**
      *in* shape:**tensor(int64)**
      *out* reshaped:**T**

      or

      *in* data:**T**
      *out* reshaped:**T**|14+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||5+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Resize|*in* X:**T**
      *in* scales:**tensor(float)**
      *out* Y:**T**

      or

      *in* X:**T1**
      *in* roi:**T2**
      *in* scales:**tensor(float)**
      *in* sizes:**tensor(int64)**
      *out* Y:**T1**|13+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(float), tensor(float16)| +|||11+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(float), tensor(float16)| +|||10+|**T** = tensor(float), tensor(float16)| +|ReverseSequence|*in* input:**T**
      *in* sequence_lens:**tensor(int64)**
      *out* Y:**T**|10+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|RoiAlign|*in* X:**T1**
      *in* rois:**T1**
      *in* batch_indices:**T2**
      *out* Y:**T1**|10+|**T1** = tensor(float), tensor(float16)
      **T2** = tensor(int32), tensor(int64)| +|Round|*in* X:**T**
      *out* Y:**T**|11+|**T** = tensor(float), tensor(float16)| +|ScaledTanh|*in* input:**T**
      *out* output:**T**|1+|**T** = tensor(float), tensor(float16)| +|Scatter|*in* data:**T**
      *in* indices:**Tind**
      *in* updates:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||9+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|ScatterElements|*in* data:**T**
      *in* indices:**Tind**
      *in* updates:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|ScatterND|*in* data:**T**
      *in* indices:**tensor(int64)**
      *in* updates:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Selu|*in* X:**T**
      *out* Y:**T**|6+|**T** = tensor(float), tensor(float16)| +|Shrink|*in* input:**T**
      *out* output:**T**|9+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|Sigmoid|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Sign|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|||9+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|Sin|*in* input:**T**
      *out* output:**T**|7+|**T** = tensor(float), tensor(float16)| +|Sinh|*in* input:**T**
      *out* output:**T**|9+|**T** = tensor(float), tensor(float16)| +|Slice|*in* data:**T**
      *in* starts:**Tind**
      *in* ends:**Tind**
      *in* axes:**Tind**
      *in* steps:**Tind**
      *out* output:**T**

      or

      *in* data:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||10+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)
      **Tind** = tensor(int32), tensor(int64)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Softmax|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||11+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|Softplus|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(float), tensor(float16)| +|Softsign|*in* input:**T**
      *out* output:**T**|1+|**T** = tensor(float), tensor(float16)| +|SpaceToDepth|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Split|*in* input:**T**
      *in* split:**T**
      *out* outputs...:**T**

      or

      *in* input:**T**
      *in* split:**tensor(int64)**
      *out* outputs:**T**

      or

      *in* input:**T**
      *out* outputs:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||2+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Sqrt|*in* X:**T**
      *out* Y:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Squeeze|*in* data:**T**
      *in* axes:**tensor(int64)**
      *out* squeezed:**T**

      or

      *in* data:**T**
      *out* squeezed:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Sub|*in* A:**T**
      *in* B:**T**
      *out* C:**T**|14+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||13+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||7+|**T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Sum|*in* data_0:**T**
      *out* sum:**T**|13+|**T** = tensor(float), tensor(float16)| +|||8+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|Tan|*in* input:**T**
      *out* output:**T**|7+|**T** = tensor(float), tensor(float16)| +|Tanh|*in* input:**T**
      *out* output:**T**|13+|**T** = tensor(float), tensor(float16)| +|||6+|**T** = tensor(float), tensor(float16)| +|ThresholdedRelu|*in* X:**T**
      *out* Y:**T**|10+|**T** = tensor(float), tensor(float16)| +|||1+|**T** = tensor(float), tensor(float16)| +|Tile|*in* input:**T**
      *in* repeats:**T1**
      *out* output:**T**

      or

      *in* input:**T**
      *in* tiles:**T**
      *in* axis:**T**
      *out* output:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||6+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|TopK|*in* X:**T**
      *in* K:**tensor(int64)**
      *out* Values:**T**
      *out* Indices:**I**

      or

      *in* X:**T**
      *out* Values:**T**
      *out* Indices:**I**|11+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||10+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**I** = tensor(int64)
      **T** = tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Transpose|*in* data:**T**
      *out* transposed:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Trilu|*in* input:**T**
      *in* k:**tensor(int64)**
      *out* output:**T**|14+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Unsqueeze|*in* data:**T**
      *in* axes:**tensor(int64)**
      *out* expanded:**T**

      or

      *in* data:**T**
      *out* expanded:**T**|13+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||11+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|||1+|**T** = tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)| +|Upsample|*in* X:**T**
      *in* scales:**tensor(float)**
      *out* Y:**T**

      or

      *in* X:**T**
      *out* Y:**T**|10+|**T** = tensor(float), tensor(float16)| +|||9+|**T** = tensor(float), tensor(float16)| +|||7+|**T** = tensor(float), tensor(float16)| +|Where|*in* condition:**B**
      *in* X:**T**
      *in* Y:**T**
      *out* output:**T**|9+|**B** = tensor(bool)
      **T** = tensor(bool), tensor(float), tensor(float16), tensor(int16), tensor(int32), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint8)| +|Xor|*in* A:**T**
      *in* B:**T**
      *out* C:**T1**|7+|**T** = tensor(bool)| +| | +| | |**Operator Domain:** *com.microsoft*|||| |Attention|*in* input:**T**
      *in* weight:**T**
      *in* bias:**T**
      *in* mask_index:**M**
      *in* past:**T**
      *in* extra_add:**T**
      *out* output:**T**
      *out* present:**T**|1+|**T** = tensor(float), tensor(float16)| |BeamSearch|*in* input_ids:**I**
      *in* max_length:**I**
      *in* min_length:**I**
      *in* num_beams:**I**
      *in* num_return_sequences:**I**
      *in* length_penalty:**T**
      *in* repetition_penalty:**T**
      *in* vocab_mask:**M**
      *in* prefix_vocab_mask:**M**
      *in* attention_mask:**I**
      *out* sequences:**I**
      *out* sequences_scores:**T**
      *out* scores:**T**|1+|**T** = tensor(float), tensor(float16)| @@ -780,6 +1086,9 @@ Do not modify directly.* |LongformerAttention|*in* input:**T**
      *in* weight:**T**
      *in* bias:**T**
      *in* mask:**T**
      *in* global_weight:**T**
      *in* global_bias:**T**
      *in* global:**G**
      *out* output:**T**|1+|**T** = tensor(float), tensor(float16)| |NGramRepeatBlock|*in* input_ids:**Tid**
      *in* scores:**T**
      *out* scores_out:**T**|1+|**T** = tensor(float)
      **Tid** = tensor(int64)| |QAttention|*in* input:**T1**
      *in* weight:**T2**
      *in* bias:**T3**
      *in* input_scale:**T3**
      *in* weight_scale:**T3**
      *in* mask_index:**T4**
      *in* input_zero_point:**T1**
      *in* weight_zero_point:**T2**
      *in* past:**T3**
      *out* output:**T3**
      *out* present:**T3**|1+|**T1** = tensor(int8)
      **T2** = tensor(int8)
      **T3** = tensor(float), tensor(float16)
      **T4** = tensor(int32)| +|QOrderedGelu|*in* X:**Q**
      *in* scale_X:**S**
      *in* scale_Y:**S**
      *out* Y:**Q**|1+|**Q** = tensor(int8)
      **S** = tensor(float)| +|QOrderedLayerNormalization|*in* X:**Q**
      *in* scale_X:**S**
      *in* scale:**F**
      *in* B:**F**
      *in* scale_Y:**S**
      *out* Y:**Q**|1+|**F** = tensor(float), tensor(float16)
      **Q** = tensor(int8)
      **S** = tensor(float)| +|QOrderedMatMul|*in* A:**Q**
      *in* scale_A:**S**
      *in* B:**Q**
      *in* scale_B:**S**
      *in* scale_Y:**S**
      *in* bias:**S**
      *in* C:**Q**
      *in* scale_C:**S**
      *out* Y:**Q**|1+|**Q** = tensor(int8)
      **S** = tensor(float)| |QuantizeLinear|*in* x:**T1**
      *in* y_scale:**T1**
      *in* y_zero_point:**T2**
      *out* y:**T2**|1+|**T1** = tensor(float16)
      **T2** = tensor(int8), tensor(uint8)| |Rfft|*in* X:**T**
      *out* Y:**T**|1+|**T** = tensor(double), tensor(float), tensor(float16)| |SkipLayerNormalization|*in* input:**T**
      *in* skip:**T**
      *in* gamma:**T**
      *in* beta:**T**
      *in* bias:**T**
      *out* output:**T**
      *out* mean:**U**
      *out* inv_std_var:**U**|1+|**T** = tensor(float), tensor(float16)| diff --git a/docs/Reduced_Operator_Kernel_build.md b/docs/Reduced_Operator_Kernel_build.md index e25f7d0478..65f9e78dad 100644 --- a/docs/Reduced_Operator_Kernel_build.md +++ b/docs/Reduced_Operator_Kernel_build.md @@ -60,7 +60,7 @@ The opset can match either the opset import for each model, or the initial ONNX e.g. if a model imports opset 12 of ONNX, all ONNX operators in that model can be listed under opset 12 for the 'ai.onnx' domain. [Netron](https://netron.app/) can be used to view an ONNX model properties to discover the opset imports. -Additionally, the ONNX operator specs for [DNN](https://github.com/onnx/onnx/blob/master/docs/Operators.md) and [traditional ML](https://github.com/onnx/onnx/blob/master/docs/Operators-ml.md) operators list the individual operator versions. +Additionally, the ONNX operator specs for [DNN](https://github.com/onnx/onnx/blob/main/docs/Operators.md) and [traditional ML](https://github.com/onnx/onnx/blob/main/docs/Operators-ml.md) operators list the individual operator versions. ### Type reduction format diff --git a/docs/Roadmap.md b/docs/Roadmap.md index eda0ffb572..728239ba46 100644 --- a/docs/Roadmap.md +++ b/docs/Roadmap.md @@ -1,11 +1,11 @@ # ONNX Runtime Roadmap -ONNX Runtime is an active, fast-paced project backed by a strong team of Microsoft engineers and data scientists along with a worldwide community of partners and contributors. This roadmap summarizes the pending investments identified by the team to continually grow +ONNX Runtime is an active, fast-paced project backed by a strong team of Microsoft engineers and data scientists along with a worldwide community of partners and contributors. This roadmap summarizes the pending investments identified by the team to continually grow ONNX Runtime as a robust, versatile, and high performance inference engine for DNN and traditional ML models. ## High Level Goals ONNX Runtime is a runtime accelerator that supports interoperable ML and DNN models based on the [ONNX](https://onnx.ai/) spec. For key technical design objectives and considerations, see [ONNX Runtime Inference High Level Design](./InferenceHighLevelDesign.md). -We recognize the challenges involved in operationalizing ML models performantly in an agile way, and we understand that high volume production services can be highly performance-sensitive and often need to support a variety of compute targets (we experience these first-hand at Microsoft across our vast array of products and services). +We recognize the challenges involved in operationalizing ML models performantly in an agile way, and we understand that high volume production services can be highly performance-sensitive and often need to support a variety of compute targets (we experience these first-hand at Microsoft across our vast array of products and services). As such, our investments are directly in support of solving those challenges, focusing on areas such as: * Platform coverage @@ -14,24 +14,24 @@ As such, our investments are directly in support of solving those challenges, fo * Model coverage * Quality and ease of use - including backwards compatibility of models (older opsets) and APIs -In addition to our OSS participation, we also internally use this technology in core products at Microsoft, with over 80 models in production providing an average of 2x+ performance improvement. +In addition to our OSS participation, we also internally use this technology in core products at Microsoft, with over 80 models in production providing an average of 2x+ performance improvement. ## Investments -In support of the high level goals outlined above, the investment areas listed below represent our active and backlog projects, -which are largely driven by community demand and anticipated usage opportunities. We will work through our prioritized backlog as -quickly as possible, and if there are any specific features or enhancements you need, we gladly welcome community contributions for -these efforts or any of the [enhancements suggested on Github](https://github.com/microsoft/onnxruntime/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement). If you have a specific suggestion or unsupported use case, please let us +In support of the high level goals outlined above, the investment areas listed below represent our active and backlog projects, +which are largely driven by community demand and anticipated usage opportunities. We will work through our prioritized backlog as +quickly as possible, and if there are any specific features or enhancements you need, we gladly welcome community contributions for +these efforts or any of the [enhancements suggested on Github](https://github.com/microsoft/onnxruntime/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement). If you have a specific suggestion or unsupported use case, please let us know by filing a [Github issue](https://github.com/microsoft/onnxruntime/issues). * **Platform coverage** - [Expanded platform compatibility](#expanded-platform-compatibility) * **Extensibility and customization** - [Accelerators and Execution Providers](#accelerators-and-execution-providers) -* **Performance** - [Continued performance optimizations](#continued-performance-optimizations) -* **Model coverage** - [Expanded model compatibility](#expanded-model-compatibility) +* **Performance** - [Continued performance optimizations](#continued-performance-optimizations) +* **Model coverage** - [Expanded model compatibility](#expanded-model-compatibility) * **Quality and ease of use** - [Increased integration with popular ML products](#increased-integration-with-popular-products) --- ### Expanded platform compatibility -ONNX Runtime already supports a wide range of architectures, platforms, and languages, and this will continue to be an active investment area to broaden the availability of the engine for varied usage. +ONNX Runtime already supports a wide range of architectures, platforms, and languages, and this will continue to be an active investment area to broaden the availability of the engine for varied usage. Additionally, we understand that lightweight devices and local applications may have constraints for package size, so there is active awareness to opportunistically minimize binary size. #### Architectures @@ -67,7 +67,7 @@ To maximize performance potential, we will be continually adding additional CUDA In addition to new execution providers, we aim to make it easy for community partners to contribute in a non-disruptive way. To support this, we are investing in improvements to the execution provider interface for easily registering new execution providers and separating out EPs from the core runtime engine. ### Continued Performance Optimizations -Performance is a key focus for ONNX Runtime. From latency to memory utilization to CPU usage, we are constantly seeking strategies to deliver the best performance. Although DNNs are rapidly driving research areas for innovation, we acknowledge that in practice, many companies and developers are still using traditional ML frameworks for reasons ranging from expertise to privacy to legality. As such, ONNX Runtime is focused on improvements and support for both DNNs and traditional ML. +Performance is a key focus for ONNX Runtime. From latency to memory utilization to CPU usage, we are constantly seeking strategies to deliver the best performance. Although DNNs are rapidly driving research areas for innovation, we acknowledge that in practice, many companies and developers are still using traditional ML frameworks for reasons ranging from expertise to privacy to legality. As such, ONNX Runtime is focused on improvements and support for both DNNs and traditional ML. #### Examples of projects the team is working on: * More quantization support @@ -79,17 +79,17 @@ Performance is a key focus for ONNX Runtime. From latency to memory utilization IoT provides growing opportunity to execute ML workloads on the edge of the network, where the data is collected. However, the devices used for ML execution have different hardware specifications. To support compatibility with this group of devices, we will invest in strategies to optimize ONNX model execution across the breadth of IoT endpoints using different hardware configurations with CPUs, GPUs and custom NN ASICs. ### Expanded model compatibility -The ONNX spec focuses on ML model interoperability rather than coverage of all operators from all frameworks. +The ONNX spec focuses on ML model interoperability rather than coverage of all operators from all frameworks. We aim to continuously improve coverage to support popular as well as new state-of-the-art models. -#### Spec coverage +#### Spec coverage As more operators are added to the ONNX spec, ONNX Runtime will provide implementations (default CPU and GPU-CUDA) of each to stay in compliance with the latest ONNX spec. This includes: * Sparse Tensor support #### Investments in popular converters -We work with the OSS and ONNX community to ensure popular frameworks can export or be converted to ONNX format. +We work with the OSS and ONNX community to ensure popular frameworks can export or be converted to ONNX format. * [PyTorch export](https://pytorch.org/docs/stable/onnx.html) * [Tensorflow-ONNX](https://github.com/onnx/tensorflow-onnx) * [Keras-ONNX](https://github.com/onnx/keras-onnx) @@ -104,8 +104,8 @@ To decrease the risk of model inferencing failures, we will improve the error ha Focusing on practicality, we take a scenario driven approach to adding additional capabilities to ONNX Runtime. ### Increased integration with popular products -We understand that data scientists and ML engineers work with many different products and toolsets to bring complex machine learning -algorithms to life through innovative user-facing applications. We want to ensure ONNX Runtime works as seamlessly as possible with +We understand that data scientists and ML engineers work with many different products and toolsets to bring complex machine learning +algorithms to life through innovative user-facing applications. We want to ensure ONNX Runtime works as seamlessly as possible with these. If you've identified any integration ideas or opportunities and have questions or need assistance, we encourage use of Github Issues as a discussion forum. Some of these products include: @@ -114,6 +114,6 @@ Some of these products include: * [ML.NET](https://docs.microsoft.com/en-us/dotnet/machine-learning/tutorials/object-detection-onnx): inference ONNX models in .NET * [PyTorch](https://pytorch.org/docs/stable/onnx.html): improve coverage for exporting trained models to ONNX * [Windows](https://docs.microsoft.com/en-us/windows/ai/windows-ml/index): run ONNX models on Windows devices using the built-in Windows ML APIs. Windows ML APIs will be included in the ONNX Runtime builds and binaries to enable Windows developers to get OS-independent updates -* [SQL Database Edge](https://docs.microsoft.com/en-us/azure/sql-database-edge/deploy-onnx): predict with ONNX models in SQL Database Edge, an optimized relational database engine geared for IoT and IoT Edge deployments +* [SQL Database Edge](https://docs.microsoft.com/en-us/azure/sql-database-edge/deploy-onnx): predict with ONNX models in SQL Database Edge, an optimized relational database engine geared for IoT and IoT Edge deployments -Have an idea or feature request? [Contribute](https://github.com/microsoft/onnxruntime/blob/master/CONTRIBUTING.md) or [let us know](https://github.com/microsoft/onnxruntime/blob/master/.github/ISSUE_TEMPLATE/feature_request.md)! +Have an idea or feature request? [Contribute](https://github.com/microsoft/onnxruntime/blob/main/CONTRIBUTING.md) or [let us know](https://github.com/microsoft/onnxruntime/blob/main/.github/ISSUE_TEMPLATE/feature_request.md)! diff --git a/docs/TVM_EP.md b/docs/TVM_EP.md index c1c0b1552b..8a34d75635 100644 --- a/docs/TVM_EP.md +++ b/docs/TVM_EP.md @@ -245,7 +245,7 @@ It is also possible to use a precompiled model. The compiled model can be obtained using the [OctoML platform](https://onnx.octoml.ai) or compiled directly (see **Support precompiled model** section in -[Sample notebook for ResNet50 inference with TVM EP](https://github.com/microsoft/onnxruntime/blob/master/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) +[Sample notebook for ResNet50 inference with TVM EP](https://github.com/microsoft/onnxruntime/blob/main/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) for more information on model compilation). In order to use the precompiled model, only need to pass two options: @@ -261,7 +261,7 @@ You can read more about these options in section [Configuration options](#config ## Samples -- [Sample notebook for ResNet50 inference with TVM EP](https://github.com/microsoft/onnxruntime/blob/master/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) +- [Sample notebook for ResNet50 inference with TVM EP](https://github.com/microsoft/onnxruntime/blob/main/docs/python/inference/notebooks/onnxruntime-tvm-tutorial.ipynb) ## Known issues - At this moment, the TVM EP has only been verified on UNIX/Linux and Windows systems. diff --git a/docs/Versioning.md b/docs/Versioning.md index d471b9cfdf..683030e928 100644 --- a/docs/Versioning.md +++ b/docs/Versioning.md @@ -22,7 +22,7 @@ models that are stamped with ONNX opset versions in the range [7-9]. ### Version matrix The [table](https://onnxruntime.ai/docs/reference/compatibility.html#onnx-opset-support) summarizes the relationship between the ONNX Runtime version and the ONNX opset version implemented in that release. Please note the backward compatibility notes above. -For more details on ONNX Release versions, see [this page](https://github.com/onnx/onnx/blob/master/docs/Versioning.md). +For more details on ONNX Release versions, see [this page](https://github.com/onnx/onnx/blob/main/docs/Versioning.md). ## Tool Compatibility A variety of tools can be used to create ONNX models. Unless otherwise noted, please use the latest released version of the tools to convert/export the ONNX model. Most tools are backwards compatible and support multiple ONNX versions. Join this with the table above to evaluate ONNX Runtime compatibility. @@ -40,4 +40,3 @@ A variety of tools can be used to create ONNX models. Unless otherwise noted, pl |[Paddle2ONNX](https://pypi.org/project/paddle2onnx/)| [Latest stable](https://github.com/PaddlePaddle/Paddle2ONNX/releases) | 1.6-1.9 | |[AutoML](https://docs.microsoft.com/en-us/azure/machine-learning/service/concept-automated-ml)|[1.0.39+](https://pypi.org/project/azureml-automl-core)|1.5| | |[1.0.33](https://pypi.org/project/azureml-automl-core/1.0.33/)|1.4| - diff --git a/docs/WinML_principles.md b/docs/WinML_principles.md index cf4d84a11c..192ad614fb 100644 --- a/docs/WinML_principles.md +++ b/docs/WinML_principles.md @@ -1,32 +1,31 @@ # Contributing to Windows ML -Window Machine Learning is a high-performance, reliable API for deploying hardware-accelerated ML inferences on Windows devices. Please visit the [Windows ML documentation](https://docs.microsoft.com/en-us/windows/ai/windows-ml/) to learn more about Windows ML. +Window Machine Learning is a high-performance, reliable API for deploying hardware-accelerated ML inferences on Windows devices. Please visit the [Windows ML documentation](https://docs.microsoft.com/en-us/windows/ai/windows-ml/) to learn more about Windows ML. ## Windows ML Base Principles **We design and optimize for all Windows devices.** - + Our goal is to provide developers with a platform that enables new experiences that run well on all Windows devices. Our design drives innovation in the DirectX ecosystem through DirectML and gives developers the confidence that their applications will work for all Windows customers. **We maintain and curate the Windows ML APIs.** - + The API is designed to ensure consistency of developer’s experience across the Windows platform. We provide long-term servicing and support, and we are committed to ensuring application’s compatibility as we evolve the API. -**Windows ML is a core component of Windows.** +**Windows ML is a core component of Windows.** The Windows ML code is packaged and distributed with each new release of Windows. To provide consumers with high-quality products, Microsoft is responsible for distributing Windows ML and related binaries as part of Windows or standalone distributable packages. ## Open for Community Contributions -We encourage community contributions to Windows ML to enhance users’ experience on Windows. We use the principles above to guide how we look at and evaluate all contributions. +We encourage community contributions to Windows ML to enhance users’ experience on Windows. We use the principles above to guide how we look at and evaluate all contributions. Ensure your feature request follows all these principles to help the review process and include information about the customer problem(s) the feature request addresses. -Note: minor issues or bugs can be addressed more quickly using the [bug/performance issue request](https://github.com/microsoft/onnxruntime/issues/new/choose) rather than feature request. +Note: minor issues or bugs can be addressed more quickly using the [bug/performance issue request](https://github.com/microsoft/onnxruntime/issues/new/choose) rather than feature request. ## Start your Feature Request -If you'd like to contribute to Windows ML and engage with the community to get feedback, please review to the contributing [process details](https://github.com/microsoft/onnxruntime/blob/master/CONTRIBUTING.md) and submit a new feature request [here](https://github.com/microsoft/onnxruntime/issues/new/choose). - +If you'd like to contribute to Windows ML and engage with the community to get feedback, please review to the contributing [process details](https://github.com/microsoft/onnxruntime/blob/main/CONTRIBUTING.md) and submit a new feature request [here](https://github.com/microsoft/onnxruntime/issues/new/choose). diff --git a/docs/onnxruntime_extensions.md b/docs/onnxruntime_extensions.md index 189108daa4..c1f4249433 100644 --- a/docs/onnxruntime_extensions.md +++ b/docs/onnxruntime_extensions.md @@ -6,7 +6,7 @@ ONNXRuntime Extensions is a comprehensive package to extend the capability of th onnxruntime-extensions supports many useful custom operators to enhance the text processing capability of ONNXRuntime, which include some widely used **string operators** and popular **tokenizers**. For custom operators supported and how to use them, please check the documentation [custom operators](https://github.com/microsoft/onnxruntime-extensions/blob/main/docs/custom_text_ops.md). ## Build ONNXRuntime with Extensions -We have supported build onnxruntime-extensions as a static library and link it into ONNXRuntime. To enable custom operators from onnxruntime-extensions, you should add argument `--use_extensions`, which will use onnxruntime-extensions from git submodule in path cmake/external/onnxruntime-extensions **by default**. +We have supported build onnxruntime-extensions as a static library and link it into ONNXRuntime. To enable custom operators from onnxruntime-extensions, you should add argument `--use_extensions`, which will use onnxruntime-extensions from git submodule in path cmake/external/onnxruntime-extensions **by default**. If you want to build ONNXRuntime with a pre-pulled onnxruntime-extensions, pass extra argument `--extensions_overridden_path `. @@ -79,7 +79,7 @@ e2e_graph = helper.make_graph( ) # ... ``` -For more usage of ONNX helper, please visit the document [Python API Overview](https://github.com/onnx/onnx/blob/master/docs/PythonAPIOverview.md). +For more usage of ONNX helper, please visit the document [Python API Overview](https://github.com/onnx/onnx/blob/main/docs/PythonAPIOverview.md). ### Run E2E Model in Python ```python @@ -96,7 +96,7 @@ sess.run(...) ``` ### Run E2E Model in JavaScript -To run E2E ONNX model in JavaScript, you need to first [prepare ONNX Runtime WebAssembly artifacts](https://github.com/microsoft/onnxruntime/tree/master/js), include the generated `ort.min.js`, and then load and run the model in JS. +To run E2E ONNX model in JavaScript, you need to first [prepare ONNX Runtime WebAssembly artifacts](https://github.com/microsoft/onnxruntime/blob/main/js), include the generated `ort.min.js`, and then load and run the model in JS. ```js // use an async context to call onnxruntime functions async function main() { @@ -122,4 +122,4 @@ async function main() { document.write(`failed to inference ONNX model: ${e}.`); } } -``` \ No newline at end of file +``` diff --git a/docs/python/ReadMeOV.rst b/docs/python/ReadMeOV.rst index b9b794fadb..3a96ced1c8 100644 --- a/docs/python/ReadMeOV.rst +++ b/docs/python/ReadMeOV.rst @@ -1,7 +1,7 @@ OpenVINO™ Execution Provider for ONNX Runtime =============================================== -`OpenVINO™ Execution Provider for ONNX Runtime `_ is a product designed for ONNX Runtime developers who want to get started with OpenVINO™ in their inferencing applications. This product delivers `OpenVINO™ `_ inline optimizations which enhance inferencing performance with minimal code modifications. +`OpenVINO™ Execution Provider for ONNX Runtime `_ is a product designed for ONNX Runtime developers who want to get started with OpenVINO™ in their inferencing applications. This product delivers `OpenVINO™ `_ inline optimizations which enhance inferencing performance with minimal code modifications. OpenVINO™ Execution Provider for ONNX Runtime accelerates inference across many `AI models `_ on a variety of Intel® hardware such as: - Intel® CPUs @@ -23,16 +23,16 @@ This package supports: - Intel® integrated GPUs - Intel® Movidius™ Vision Processing Units (VPUs). -Please Note for VAD-M use Docker installation / Build from Source for Linux. +Please Note for VAD-M use Docker installation / Build from Source for Linux. ``pip3 install onnxruntime-openvino==1.12.0`` -Please install OpenVINO™ PyPi Package separately for Windows. +Please install OpenVINO™ PyPi Package separately for Windows. For installation instructions on Windows please refer to `OpenVINO™ Execution Provider for ONNX Runtime for Windows `_. **OpenVINO™ Execution Provider for ONNX Runtime** Linux Wheels comes with pre-built libraries of OpenVINO™ version 2022.1.0 eliminating the need to install OpenVINO™ separately. The OpenVINO™ libraries are prebuilt with CXX11_ABI flag set to 0. -The package also includes module that is used by torch-ort-inference to accelerate inference for PyTorch models with OpenVINO Execution Provider. +The package also includes module that is used by torch-ort-inference to accelerate inference for PyTorch models with OpenVINO Execution Provider. See `torch-ort-inference `_ for more details. For more details on build and installation please refer to `Build `_. @@ -53,21 +53,21 @@ To see what you can do with **OpenVINO™ Execution Provider for ONNX Runtime**, Docker Support ^^^^^^^^^^^^^^ -The latest OpenVINO™ EP docker image can be downloaded from DockerHub. +The latest OpenVINO™ EP docker image can be downloaded from DockerHub. For more details see `Docker ReadMe `_. Prebuilt Images ^^^^^^^^^^^^^^^^ -- Please find prebuilt docker images for Intel® CPU and Intel® iGPU on OpenVINO™ Execution Provider `Release Page `_. +- Please find prebuilt docker images for Intel® CPU and Intel® iGPU on OpenVINO™ Execution Provider `Release Page `_. License ^^^^^^^^ -**OpenVINO™ Execution Provider for ONNX Runtime** is licensed under `MIT `_. +**OpenVINO™ Execution Provider for ONNX Runtime** is licensed under `MIT `_. By contributing to the project, you agree to the license and copyright terms therein -and release your contribution under these terms. +and release your contribution under these terms. Support ^^^^^^^^ diff --git a/docs/python/inference/api_summary.rst b/docs/python/inference/api_summary.rst index 12d17bafd3..c6d1f4868e 100644 --- a/docs/python/inference/api_summary.rst +++ b/docs/python/inference/api_summary.rst @@ -17,7 +17,7 @@ Load and run a model -------------------- InferenceSession is the main class of ONNX Runtime. It is used to load and run an ONNX model, -as well as specify environment and application configuration options. +as well as specify environment and application configuration options. .. code-block:: python @@ -65,7 +65,7 @@ Data on CPU ^^^^^^^^^^^ On CPU (the default), OrtValues can be mapped to and from native Python data structures: numpy arrays, dictionaries and lists of -numpy arrays. +numpy arrays. .. code-block:: python @@ -95,15 +95,15 @@ this called `IOBinding`. To use the `IOBinding` feature, replace `InferenceSession.run()` with `InferenceSession.run_with_iobinding()`. -A graph is executed on a device other than CPU, for instance CUDA. Users can +A graph is executed on a device other than CPU, for instance CUDA. Users can use IOBinding to copy the data onto the GPU. .. code-block:: python - # X is numpy array on cpu + # X is numpy array on cpu session = onnxruntime.InferenceSession('model.onnx', providers=['CUDAExecutionProvider', 'CPUExecutionProvider'])) io_binding = session.io_binding() - # OnnxRuntime will copy the data over to the CUDA device if 'input' is consumed by nodes on the CUDA device + # OnnxRuntime will copy the data over to the CUDA device if 'input' is consumed by nodes on the CUDA device io_binding.bind_cpu_input('input', X) io_binding.bind_output('output') session.run_with_iobinding(io_binding) @@ -122,7 +122,7 @@ The input data is on a device, users directly use the input. The output data is session.run_with_iobinding(io_binding) Y = io_binding.copy_outputs_to_cpu()[0] -The input data and output data are both on a device, users directly use the input and also place output on the device. +The input data and output data are both on a device, users directly use the input and also place output on the device. .. code-block:: python @@ -284,7 +284,7 @@ Backend In addition to the regular API which is optimized for performance and usability,  *ONNX Runtime* also implements the -`ONNX backend API `_ +`ONNX backend API `_ for verification of *ONNX* specification conformance. The following functions are supported: diff --git a/docs/python/inference/examples/plot_backend.py b/docs/python/inference/examples/plot_backend.py index b441012e63..ecfc17175d 100644 --- a/docs/python/inference/examples/plot_backend.py +++ b/docs/python/inference/examples/plot_backend.py @@ -8,8 +8,8 @@ ONNX Runtime Backend for ONNX ============================= -*ONNX Runtime* extends the -`onnx backend API `_ +*ONNX Runtime* extends the +`onnx backend API `_ to run predictions using this runtime. Let's use the API to compute the prediction of a simple logistic regression model. diff --git a/docs/python/inference/examples/plot_load_and_predict.py b/docs/python/inference/examples/plot_load_and_predict.py index 44e343163a..09d7c9cdb4 100644 --- a/docs/python/inference/examples/plot_load_and_predict.py +++ b/docs/python/inference/examples/plot_load_and_predict.py @@ -19,7 +19,7 @@ from onnxruntime.datasets import get_example ######################### # Let's load a very simple model. -# The model is available on github `onnx...test_sigmoid `_. +# The model is available on github `onnx...test_sigmoid `_. example1 = get_example("sigmoid.onnx") sess = rt.InferenceSession(example1, providers=rt.get_available_providers()) diff --git a/docs/python/inference/examples/plot_pipeline.py b/docs/python/inference/examples/plot_pipeline.py index 366aadee1c..05dcbdb25b 100644 --- a/docs/python/inference/examples/plot_pipeline.py +++ b/docs/python/inference/examples/plot_pipeline.py @@ -6,7 +6,7 @@ Draw a pipeline =============== There is no other way to look into one model stored -in ONNX format than looking into its node with +in ONNX format than looking into its node with *onnx*. This example demonstrates how to draw a model and to retrieve it in *json* format. @@ -34,7 +34,7 @@ print(model) ################################# # Draw a model with ONNX # ++++++++++++++++++++++ -# We use `net_drawer.py `_ +# We use `net_drawer.py `_ # included in *onnx* package. # We use *onnx* to load the model # in a different way than before. diff --git a/docs/python/inference/notebooks/onnxruntime-nuphar-tutorial.ipynb b/docs/python/inference/notebooks/onnxruntime-nuphar-tutorial.ipynb index 231a801c17..3eef63dea0 100644 --- a/docs/python/inference/notebooks/onnxruntime-nuphar-tutorial.ipynb +++ b/docs/python/inference/notebooks/onnxruntime-nuphar-tutorial.ipynb @@ -17,7 +17,7 @@ "\n", "This example shows how to accelerate model inference using Nuphar, an execution provider that leverages just-in-time compilation to generate optimized executables.\n", "\n", - "For more background about Nuphar, please check [Nuphar-ExecutionProvider.md](https://github.com/microsoft/onnxruntime/blob/master/docs/execution_providers/Nuphar-ExecutionProvider.md) and its [build instructions](https://www.onnxruntime.ai/docs/how-to/build.html#nuphar).\n", + "For more background about Nuphar, please check [Nuphar-ExecutionProvider.md](https://github.com/microsoft/onnxruntime/blob/main/docs/execution_providers/Nuphar-ExecutionProvider.md) and its [build instructions](https://www.onnxruntime.ai/docs/how-to/build.html#nuphar).\n", "\n", "#### Tutorial Roadmap:\n", "1. Prerequistes\n", @@ -36,7 +36,7 @@ "## 1. Prerequistes\n", "Please make sure you have installed following Python packages. Besides, C++ compiler/linker is required for ahead-of-time compilation. Please make sure you have g++ if running on Linux, or Visual Studio 2017 on Windows.\n", "\n", - "For simplicity, you may use [Nuphar docker image](https://github.com/microsoft/onnxruntime/blob/master/dockerfiles/README.md) from Microsoft Container Registry.\n" + "For simplicity, you may use [Nuphar docker image](https://github.com/microsoft/onnxruntime/blob/main/dockerfiles/README.md) from Microsoft Container Registry.\n" ] }, { @@ -1199,4 +1199,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/include/onnxruntime/core/common/common.h b/include/onnxruntime/core/common/common.h index c405bcacca..501634bf32 100644 --- a/include/onnxruntime/core/common/common.h +++ b/include/onnxruntime/core/common/common.h @@ -42,10 +42,6 @@ namespace onnxruntime { using TimePoint = std::chrono::high_resolution_clock::time_point; -// Using statements for common classes that we refer to in ONNXRuntime very often. -// TODO(Task:137) Remove 'using' statements from header files -using common::Status; - #ifdef _WIN32 #define ORT_UNUSED_PARAMETER(x) (x) #else diff --git a/include/onnxruntime/core/common/eigen_common_wrapper.h b/include/onnxruntime/core/common/eigen_common_wrapper.h index 20a2757466..4515f80ffd 100644 --- a/include/onnxruntime/core/common/eigen_common_wrapper.h +++ b/include/onnxruntime/core/common/eigen_common_wrapper.h @@ -34,6 +34,13 @@ #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif +// cmake/external/eigen\Eigen/CXX11/src/Tensor/TensorTrace.h:130:9: +// error: variable 'num_distinct_reduce_dims' set but not used [-Werror,-Wunused-but-set-variable] +// int num_distinct_reduce_dims = 0; +#ifdef HAS_UNUSED_BUT_SET_VARIABLE +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + #elif defined(_MSC_VER) // build\windows\debug\external\eigen3\unsupported\eigen\cxx11\src/Tensor/Tensor.h(76): // warning C4554: '&': check operator precedence for possible error; use parentheses to clarify precedence diff --git a/include/onnxruntime/core/common/profiler_common.h b/include/onnxruntime/core/common/profiler_common.h index ec35e111d5..a2f31442b1 100644 --- a/include/onnxruntime/core/common/profiler_common.h +++ b/include/onnxruntime/core/common/profiler_common.h @@ -5,6 +5,7 @@ #include "core/common/common.h" #include +#include namespace onnxruntime { namespace profiling { @@ -13,6 +14,7 @@ enum EventCategory { SESSION_EVENT = 0, NODE_EVENT, KERNEL_EVENT, + API_EVENT, EVENT_CATEGORY_MAX }; @@ -20,7 +22,8 @@ enum EventCategory { static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { "Session", "Node", - "Kernel"}; + "Kernel", + "Api"}; // Timing record for all events. struct EventRecord { @@ -58,5 +61,9 @@ class EpProfiler { virtual void Stop(uint64_t){}; // called after op stop, accept an id as argument to identify the op }; -} //namespace profiling -} //namespace onnxruntime \ No newline at end of file +// Demangle C++ symbols +std::string demangle(const char* name); +std::string demangle(const std::string& name); + +} // namespace profiling +} // namespace onnxruntime diff --git a/include/onnxruntime/core/common/status.h b/include/onnxruntime/core/common/status.h index b7c36df2b2..bcf058886e 100644 --- a/include/onnxruntime/core/common/status.h +++ b/include/onnxruntime/core/common/status.h @@ -188,4 +188,8 @@ inline std::ostream& operator<<(std::ostream& out, const Status& status) { } } // namespace common + +// make Status directly available in the onnxruntime namespace as it is widely used +using common::Status; + } // namespace onnxruntime diff --git a/include/onnxruntime/core/common/string_helper.h b/include/onnxruntime/core/common/string_helper.h index ea6ff75f64..1304303132 100644 --- a/include/onnxruntime/core/common/string_helper.h +++ b/include/onnxruntime/core/common/string_helper.h @@ -2,16 +2,10 @@ // Licensed under the MIT License. #pragma once +#include -#include "core/session/onnxruntime_c_api.h" - +// forward declaration +struct OrtAllocator; namespace onnxruntime { -#if defined(USE_TENSORRT) || defined(USE_CUDA) -static char* StrDup(const std::string& str, _Inout_ OrtAllocator* allocator) { - char* output_string = reinterpret_cast(allocator->Alloc(allocator, str.size() + 1)); - memcpy(output_string, str.c_str(), str.size()); - output_string[str.size()] = '\0'; - return output_string; -} -#endif +char* StrDup(const std::string& str, OrtAllocator* allocator); } // namespace onnxruntime diff --git a/include/onnxruntime/core/framework/ortmemoryinfo.h b/include/onnxruntime/core/framework/ortmemoryinfo.h index ff0c90ac84..32b7aab7c4 100644 --- a/include/onnxruntime/core/framework/ortmemoryinfo.h +++ b/include/onnxruntime/core/framework/ortmemoryinfo.h @@ -48,7 +48,7 @@ struct OrtMemoryInfo { auto h = std::hash()(alloc_type); HashCombine(std::hash()(mem_type), h); HashCombine(std::hash()(id), h); - HashCombine(std::hash()(name), h); + HashCombine(std::hash()(name), h); return h; } diff --git a/include/onnxruntime/core/framework/run_options.h b/include/onnxruntime/core/framework/run_options.h index b3edd3889e..76cb89a8e7 100644 --- a/include/onnxruntime/core/framework/run_options.h +++ b/include/onnxruntime/core/framework/run_options.h @@ -12,7 +12,7 @@ * Configuration information for a Run call. */ struct OrtRunOptions { - /// Log severity. See https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/common/logging/severity.h + /// Log severity. See https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/common/logging/severity.h /// Default = -1 (use the log severity from the InferenceSession that the Run is for). int run_log_severity_level = -1; int run_log_verbosity_level = 0; ///< VLOG level if debug build and run_log_severity_level is 0 (VERBOSE). diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h index 8f81ce14dd..8e930ce5a8 100644 --- a/include/onnxruntime/core/graph/graph.h +++ b/include/onnxruntime/core/graph/graph.h @@ -18,6 +18,7 @@ #if !defined(ORT_MINIMAL_BUILD) #include "onnx/defs/schema.h" +#include "core/common/inlined_containers.h" #else #include "onnx/defs/data_type_utils.h" #endif @@ -1092,13 +1093,6 @@ class Graph { */ Node& BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::string& fused_node_name); - /** - If we have BeginFuseSubGraph, but somehow hit errors, such as Compile of an EP failed on thesub_graph. - We can call CancelFuseSubGraph to undo the changes of BeginFuseSubGraph - @param fused_node The fused node and it's function body to be removed from the graph - */ - void CancelFuseSubGraph(const Node& fused_node); - void FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node); #endif @@ -1605,6 +1599,9 @@ class Graph { // for the fused kernel. I really don't like it. but for short-term solution, let's host // those schemas here. InlinedVector> fused_schemas_containers_; + // in some case, a fused sub-graph will happens multiple times in one model, we use a map + // to store reusable-schema in lookup. + InlinedHashMap> reusable_fused_schema_map_; #endif // !defined(ORT_MINIMAL_BUILD) // Graph nodes. diff --git a/include/onnxruntime/core/graph/indexed_sub_graph.h b/include/onnxruntime/core/graph/indexed_sub_graph.h index 639384f6cc..935ada88c6 100644 --- a/include/onnxruntime/core/graph/indexed_sub_graph.h +++ b/include/onnxruntime/core/graph/indexed_sub_graph.h @@ -52,9 +52,17 @@ struct IndexedSubGraph { /** Nodes covered by this subgraph. The NodeIndex values are from the parent Graph.*/ std::vector nodes; - // use an existing schema instead of generating one when fusing nodes using the MetaDef. + enum class SourceOfSchema : uint8_t { + CREATE, /// create new schema from info in IndexedSubGraph instance. + /// schema instance will not be re-usable. + REUSE_OR_CREATE, /// re-use existing dynamically created schema with matching domain+name. + /// create re-usable schema if one is not found. + EXISTING, /// use existing statically registered schema. + /// e.g. domain+name matches ONNX or contrib op domain+op_type+opset. + }; + // Either using an existing schema or generating reusable one when fusing nodes using the MetaDef. // MetaDef.domain + MetaDef.name => the domain.op_type that a schema must exist for with a valid since_version. - bool use_existing_schema{false}; + SourceOfSchema schema_source{SourceOfSchema::CREATE}; /** Set the meta definition needed to represent this subgraph as a FunctionProto It's needed IF AND ONLY IF there are multiple indexes contained in #nodes. */ diff --git a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h index e0e0711629..ed94b14578 100644 --- a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h +++ b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h @@ -449,7 +449,7 @@ class RunQueue { // PushBack adds w at the end of the queue. // If queue is full returns w, otherwise returns default-constructed Work. Work PushBack(Work w) { - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); unsigned back = back_.load(std::memory_order_relaxed); Elem& e = array_[(back - 1) & kMask]; ElemState s = e.state.load(std::memory_order_relaxed); @@ -469,7 +469,7 @@ class RunQueue { // with w_idx. Typically the tag will be a per-thread ID to distinguish work // submitted from different threads. PushResult PushBackWithTag(Work w, Tag tag, unsigned& w_idx) { - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); unsigned back = back_.load(std::memory_order_relaxed); w_idx = (back - 1) & kMask; Elem& e = array_[w_idx]; @@ -490,7 +490,7 @@ class RunQueue { Work PopBack() { if (Empty()) return Work(); - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); unsigned back; Elem* e; ElemState s; @@ -532,7 +532,7 @@ class RunQueue { bool RevokeWithTag(Tag tag, unsigned w_idx) { bool revoked = false; - std::unique_lock lock(mutex_); + std::lock_guard lock(mutex_); Elem& e = array_[w_idx]; ElemState s = e.state.load(std::memory_order_relaxed); diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index e9a04fc67f..625c42ed42 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3494,7 +3494,7 @@ struct OrtApi { * * \since Version 1.13 */ - const OrtTrainingApi*(ORT_API_CALL* GetTrainingApi)(uint32_t version)NO_EXCEPTION ORT_ALL_ARGS_NONNULL; + const OrtTrainingApi*(ORT_API_CALL* GetTrainingApi)(uint32_t version) NO_EXCEPTION; #ifdef __cplusplus OrtApi(const OrtApi&)=delete; // Prevent users from accidentally copying the API structure, it should always be passed as a pointer diff --git a/java/src/main/java/ai/onnxruntime/MapInfo.java b/java/src/main/java/ai/onnxruntime/MapInfo.java index 5527931395..5a2ac072f3 100644 --- a/java/src/main/java/ai/onnxruntime/MapInfo.java +++ b/java/src/main/java/ai/onnxruntime/MapInfo.java @@ -4,6 +4,8 @@ */ package ai.onnxruntime; +import ai.onnxruntime.TensorInfo.OnnxTensorType; + /** Describes an {@link OnnxMap} object or output node. */ public class MapInfo implements ValueInfo { @@ -42,6 +44,21 @@ public class MapInfo implements ValueInfo { this.valueType = valueType; } + /** + * Construct a MapInfo with the specified size, key type and value type. + * + *

      Called from JNI. + * + * @param size The size. + * @param keyTypeInt The int representing the {@link OnnxTensorType} of the keys. + * @param valueTypeInt The int representing the {@link OnnxTensorType} of the values. + */ + MapInfo(int size, int keyTypeInt, int valueTypeInt) { + this.size = size; + this.keyType = OnnxJavaType.mapFromOnnxTensorType(OnnxTensorType.mapFromInt(keyTypeInt)); + this.valueType = OnnxJavaType.mapFromOnnxTensorType(OnnxTensorType.mapFromInt(valueTypeInt)); + } + @Override public String toString() { String initial = size == -1 ? "MapInfo(size=UNKNOWN" : "MapInfo(size=" + size; diff --git a/java/src/main/java/ai/onnxruntime/OnnxTensor.java b/java/src/main/java/ai/onnxruntime/OnnxTensor.java index f71d26c8bf..653a3c9be9 100644 --- a/java/src/main/java/ai/onnxruntime/OnnxTensor.java +++ b/java/src/main/java/ai/onnxruntime/OnnxTensor.java @@ -90,14 +90,14 @@ public class OnnxTensor implements OnnxValue { case BOOL: return getBool(OnnxRuntime.ortApiHandle, nativeHandle); case STRING: - return getString(OnnxRuntime.ortApiHandle, nativeHandle, allocatorHandle); + return getString(OnnxRuntime.ortApiHandle, nativeHandle); case UNKNOWN: default: throw new OrtException("Extracting the value of an invalid Tensor."); } } else { Object carrier = info.makeCarrier(); - getArray(OnnxRuntime.ortApiHandle, nativeHandle, allocatorHandle, carrier); + getArray(OnnxRuntime.ortApiHandle, nativeHandle, carrier); if ((info.type == OnnxJavaType.STRING) && (info.shape.length != 1)) { // We read the strings out from native code in a flat array and then reshape // to the desired output shape. @@ -284,13 +284,12 @@ public class OnnxTensor implements OnnxValue { private native long getLong(long apiHandle, long nativeHandle, int onnxType) throws OrtException; - private native String getString(long apiHandle, long nativeHandle, long allocatorHandle) - throws OrtException; + private native String getString(long apiHandle, long nativeHandle) throws OrtException; private native boolean getBool(long apiHandle, long nativeHandle) throws OrtException; - private native void getArray( - long apiHandle, long nativeHandle, long allocatorHandle, Object carrier) throws OrtException; + private native void getArray(long apiHandle, long nativeHandle, Object carrier) + throws OrtException; private native void close(long apiHandle, long nativeHandle); diff --git a/java/src/main/java/ai/onnxruntime/OrtEnvironment.java b/java/src/main/java/ai/onnxruntime/OrtEnvironment.java index b5f1fb21af..090fe34a45 100644 --- a/java/src/main/java/ai/onnxruntime/OrtEnvironment.java +++ b/java/src/main/java/ai/onnxruntime/OrtEnvironment.java @@ -15,8 +15,8 @@ import java.util.logging.Logger; * *

      There can be at most one OrtEnvironment object created in a JVM lifetime. This class * implements {@link AutoCloseable} as before for backwards compatibility with 1.10 and earlier, but - * the {@link #close} method is a no-op. The environment is closed by a JVM shutdown hook registered - * on construction. + * the close method is a no-op. The environment is closed by a JVM shutdown hook registered on + * construction. */ public final class OrtEnvironment implements AutoCloseable { diff --git a/java/src/main/java/ai/onnxruntime/OrtProvider.java b/java/src/main/java/ai/onnxruntime/OrtProvider.java index cf5202c432..21ad99fb8b 100644 --- a/java/src/main/java/ai/onnxruntime/OrtProvider.java +++ b/java/src/main/java/ai/onnxruntime/OrtProvider.java @@ -23,7 +23,8 @@ public enum OrtProvider { ACL("ACLExecutionProvider"), ARM_NN("ArmNNExecutionProvider"), ROCM("ROCMExecutionProvider"), - CORE_ML("CoreMLExecutionProvider"); + CORE_ML("CoreMLExecutionProvider"), + XNNPACK("XnnpackExecutionProvider"); private static final Map valueMap = new HashMap<>(values().length); diff --git a/java/src/main/java/ai/onnxruntime/OrtSession.java b/java/src/main/java/ai/onnxruntime/OrtSession.java index adc38d3aa5..0f4bcc639a 100644 --- a/java/src/main/java/ai/onnxruntime/OrtSession.java +++ b/java/src/main/java/ai/onnxruntime/OrtSession.java @@ -987,6 +987,27 @@ public class OrtSession implements AutoCloseable { addCoreML(OnnxRuntime.ortApiHandle, nativeHandle, OrtFlags.aggregateToInt(flags)); } + /** + * Adds Xnnpack as an execution backend. Needs to list all options here if a new option + * supported. current supported options: {} + * + * @param providerOptions options pass to XNNPACK EP for initialization. + * @throws OrtException If there was an error in native code. + */ + public void addXnnpack(Map providerOptions) throws OrtException { + checkClosed(); + String[] providerOptionKey = new String[providerOptions.size()]; + String[] providerOptionVal = new String[providerOptions.size()]; + int i = 0; + for (Map.Entry entry : providerOptions.entrySet()) { + providerOptionKey[i] = entry.getKey(); + providerOptionVal[i] = entry.getValue(); + i++; + } + addExecutionProvider( + OnnxRuntime.ortApiHandle, nativeHandle, "XNNPACK", providerOptionKey, providerOptionVal); + } + private native void setExecutionMode(long apiHandle, long nativeHandle, int mode) throws OrtException; @@ -1098,6 +1119,14 @@ public class OrtSession implements AutoCloseable { private native void addCoreML(long apiHandle, long nativeHandle, int coreMLFlags) throws OrtException; + + private native void addExecutionProvider( + long apiHandle, + long nativeHandle, + String epName, + String[] providerOptionKey, + String[] providerOptionVal) + throws OrtException; } /** Used to control logging and termination of a call to {@link OrtSession#run}. */ diff --git a/java/src/main/java/ai/onnxruntime/SequenceInfo.java b/java/src/main/java/ai/onnxruntime/SequenceInfo.java index a417634b72..5497766546 100644 --- a/java/src/main/java/ai/onnxruntime/SequenceInfo.java +++ b/java/src/main/java/ai/onnxruntime/SequenceInfo.java @@ -4,6 +4,8 @@ */ package ai.onnxruntime; +import ai.onnxruntime.TensorInfo.OnnxTensorType; + /** Describes an {@link OnnxSequence}, including it's element type if known. */ public class SequenceInfo implements ValueInfo { @@ -35,6 +37,23 @@ public class SequenceInfo implements ValueInfo { this.mapInfo = null; } + /** + * Construct a sequence of known length, with the specified type. This sequence does not contain + * maps. + * + *

      Called from JNI. + * + * @param length The length of the sequence. + * @param sequenceTypeInt The element type int of the sequence mapped from {@link OnnxTensorType}. + */ + SequenceInfo(int length, int sequenceTypeInt) { + this.length = length; + this.sequenceType = + OnnxJavaType.mapFromOnnxTensorType(OnnxTensorType.mapFromInt(sequenceTypeInt)); + this.sequenceOfMaps = false; + this.mapInfo = null; + } + /** * Construct a sequence of known length containing maps. * diff --git a/java/src/main/java/ai/onnxruntime/TensorInfo.java b/java/src/main/java/ai/onnxruntime/TensorInfo.java index ead90d83bc..4a7a3b833b 100644 --- a/java/src/main/java/ai/onnxruntime/TensorInfo.java +++ b/java/src/main/java/ai/onnxruntime/TensorInfo.java @@ -120,6 +120,20 @@ public class TensorInfo implements ValueInfo { this.onnxType = onnxType; } + /** + * Constructs a TensorInfo with the specified shape and native type int. + * + *

      Called from JNI. + * + * @param shape The tensor shape. + * @param typeInt The native type int. + */ + TensorInfo(long[] shape, int typeInt) { + this.shape = shape; + this.onnxType = OnnxTensorType.mapFromInt(typeInt); + this.type = OnnxJavaType.mapFromOnnxTensorType(this.onnxType); + } + /** * Get a copy of the tensor's shape. * diff --git a/java/src/main/native/OrtJniUtil.c b/java/src/main/native/OrtJniUtil.c index 55f26f35b4..a670179a0e 100644 --- a/java/src/main/native/OrtJniUtil.c +++ b/java/src/main/native/OrtJniUtil.c @@ -8,7 +8,8 @@ jint JNI_OnLoad(JavaVM *vm, void *reserved) { // To silence unused-parameter error. - // This function must exist according to the JNI spec, but the arguments aren't necessary for the library to request a specific version. + // This function must exist according to the JNI spec, but the arguments aren't necessary for the library + // to request a specific version. (void)vm; (void) reserved; // Requesting 1.6 to support Android. Will need to be bumped to a later version to call interface default methods // from native code, or to access other new Java features. @@ -213,289 +214,271 @@ typedef union FP32 { float floatVal; } FP32; -jfloat convertHalfToFloat(uint16_t half) { +jfloat convertHalfToFloat(const uint16_t half) { FP32 output; output.intVal = (((half&0x8000)<<16) | (((half&0x7c00)+0x1C000)<<13) | ((half&0x03FF)<<13)); return output.floatVal; } -jobject convertToValueInfo(JNIEnv *jniEnv, const OrtApi * api, OrtTypeInfo * info) { - ONNXType type; - checkOrtStatus(jniEnv,api,api->GetOnnxTypeFromTypeInfo(info,&type)); +jobject convertToValueInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtTypeInfo * info) { + ONNXType type = ONNX_TYPE_UNKNOWN; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetOnnxTypeFromTypeInfo(info, &type)); + if (code != ORT_OK) { + return NULL; + } - switch (type) { - case ONNX_TYPE_TENSOR: { - const OrtTensorTypeAndShapeInfo* tensorInfo; - checkOrtStatus(jniEnv,api,api->CastTypeInfoToTensorInfo(info,&tensorInfo)); - return convertToTensorInfo(jniEnv, api, (const OrtTensorTypeAndShapeInfo *) tensorInfo); - } - case ONNX_TYPE_SEQUENCE: { - const OrtSequenceTypeInfo* sequenceInfo; - checkOrtStatus(jniEnv,api,api->CastTypeInfoToSequenceTypeInfo(info,&sequenceInfo)); - return convertToSequenceInfo(jniEnv, api, sequenceInfo); - } - case ONNX_TYPE_MAP: { - const OrtMapTypeInfo* mapInfo; - checkOrtStatus(jniEnv,api,api->CastTypeInfoToMapTypeInfo(info,&mapInfo)); - return convertToMapInfo(jniEnv, api, mapInfo); - } - case ONNX_TYPE_UNKNOWN: - case ONNX_TYPE_OPAQUE: - case ONNX_TYPE_SPARSETENSOR: - default: { - throwOrtException(jniEnv,convertErrorCode(ORT_NOT_IMPLEMENTED),"Invalid ONNXType found."); - return NULL; - } + switch (type) { + case ONNX_TYPE_TENSOR: { + const OrtTensorTypeAndShapeInfo* tensorInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->CastTypeInfoToTensorInfo(info, &tensorInfo)); + if (code == ORT_OK) { + return convertToTensorInfo(jniEnv, api, tensorInfo); + } else { + return NULL; + } } + case ONNX_TYPE_SEQUENCE: { + const OrtSequenceTypeInfo* sequenceInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->CastTypeInfoToSequenceTypeInfo(info, &sequenceInfo)); + if (code == ORT_OK) { + return convertToSequenceInfo(jniEnv, api, sequenceInfo); + } else { + return NULL; + } + } + case ONNX_TYPE_MAP: { + const OrtMapTypeInfo* mapInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->CastTypeInfoToMapTypeInfo(info, &mapInfo)); + if (code == ORT_OK) { + return convertToMapInfo(jniEnv, api, mapInfo); + } else { + return NULL; + } + } + case ONNX_TYPE_UNKNOWN: + case ONNX_TYPE_OPAQUE: + case ONNX_TYPE_SPARSETENSOR: + default: { + throwOrtException(jniEnv,convertErrorCode(ORT_NOT_IMPLEMENTED),"Invalid ONNXType found."); + return NULL; + } + } } jobject convertToTensorInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtTensorTypeAndShapeInfo * info) { - // Extract the information from the info struct. - ONNXTensorElementDataType onnxType; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(info,&onnxType)); - size_t numDim; - checkOrtStatus(jniEnv,api,api->GetDimensionsCount(info,&numDim)); - //printf("numDim %d\n",numDim); - int64_t* dimensions = (int64_t*) malloc(sizeof(int64_t)*numDim); - checkOrtStatus(jniEnv,api,api->GetDimensions(info, dimensions, numDim)); - jint onnxTypeInt = convertFromONNXDataFormat(onnxType); + // Extract the information from the info struct. + ONNXTensorElementDataType onnxType; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetTensorElementType(info, &onnxType)); + if (code != ORT_OK) { + return NULL; + } + size_t numDim = 0; + code = checkOrtStatus(jniEnv, api, api->GetDimensionsCount(info, &numDim)); + if (code != ORT_OK) { + return NULL; + } + //printf("numDim %d\n",numDim); + int64_t* dimensions = (int64_t*) malloc(sizeof(int64_t)*numDim); + code = checkOrtStatus(jniEnv, api, api->GetDimensions(info, dimensions, numDim)); + if (code != ORT_OK) { + free((void*) dimensions); + return NULL; + } + jint onnxTypeInt = convertFromONNXDataFormat(onnxType); - // Create the long array for the shape. - jlongArray shape = (*jniEnv)->NewLongArray(jniEnv, safecast_size_t_to_jsize(numDim)); - (*jniEnv)->SetLongArrayRegion(jniEnv, shape, 0, safecast_size_t_to_jsize(numDim), (jlong*)dimensions); - // Free the dimensions array - free(dimensions); - dimensions = NULL; + // Create the long array for the shape. + jlongArray shape = (*jniEnv)->NewLongArray(jniEnv, safecast_size_t_to_jsize(numDim)); + (*jniEnv)->SetLongArrayRegion(jniEnv, shape, 0, safecast_size_t_to_jsize(numDim), (jlong*)dimensions); + // Free the dimensions array + free(dimensions); + dimensions = NULL; - // Create the ONNXTensorType enum - char *onnxTensorTypeClassName = "ai/onnxruntime/TensorInfo$OnnxTensorType"; - jclass clazz = (*jniEnv)->FindClass(jniEnv, onnxTensorTypeClassName); - jmethodID onnxTensorTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,clazz, "mapFromInt", "(I)Lai/onnxruntime/TensorInfo$OnnxTensorType;"); - jobject onnxTensorTypeJava = (*jniEnv)->CallStaticObjectMethod(jniEnv,clazz,onnxTensorTypeMapFromInt,onnxTypeInt); - //printf("ONNXTensorType class %p, methodID %p, object %p\n",clazz,onnxTensorTypeMapFromInt,onnxTensorTypeJava); - - // Create the ONNXJavaType enum - char *javaDataTypeClassName = "ai/onnxruntime/OnnxJavaType"; - clazz = (*jniEnv)->FindClass(jniEnv, javaDataTypeClassName); - jmethodID javaDataTypeMapFromONNXTensorType = (*jniEnv)->GetStaticMethodID(jniEnv,clazz, "mapFromOnnxTensorType", "(Lai/onnxruntime/TensorInfo$OnnxTensorType;)Lai/onnxruntime/OnnxJavaType;"); - jobject javaDataType = (*jniEnv)->CallStaticObjectMethod(jniEnv,clazz,javaDataTypeMapFromONNXTensorType,onnxTensorTypeJava); - //printf("JavaDataType class %p, methodID %p, object %p\n",clazz,javaDataTypeMapFromONNXTensorType,javaDataType); - - // Create the TensorInfo object - char *tensorInfoClassName = "ai/onnxruntime/TensorInfo"; - clazz = (*jniEnv)->FindClass(jniEnv, tensorInfoClassName); - jmethodID tensorInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,clazz, "", "([JLai/onnxruntime/OnnxJavaType;Lai/onnxruntime/TensorInfo$OnnxTensorType;)V"); - //printf("TensorInfo class %p, methodID %p\n",clazz,tensorInfoConstructor); - jobject tensorInfo = (*jniEnv)->NewObject(jniEnv, clazz, tensorInfoConstructor, shape, javaDataType, onnxTensorTypeJava); - return tensorInfo; + // Create the TensorInfo object + static const char *tensorInfoClassName = "ai/onnxruntime/TensorInfo"; + jclass clazz = (*jniEnv)->FindClass(jniEnv, tensorInfoClassName); + jmethodID tensorInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,clazz, "", "([JI)V"); + //printf("TensorInfo class %p, methodID %p\n",clazz,tensorInfoConstructor); + jobject tensorInfo = (*jniEnv)->NewObject(jniEnv, clazz, tensorInfoConstructor, shape, onnxTypeInt); + return tensorInfo; } jobject convertToMapInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtMapTypeInfo * info) { - // Create the java methods we need to call. - // Get the ONNXTensorType enum static method - char *onnxTensorTypeClassName = "ai/onnxruntime/TensorInfo$OnnxTensorType"; - jclass onnxTensorTypeClazz = (*jniEnv)->FindClass(jniEnv, onnxTensorTypeClassName); - jmethodID onnxTensorTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,onnxTensorTypeClazz, "mapFromInt", "(I)Lai/onnxruntime/TensorInfo$OnnxTensorType;"); + // Extract the key type + ONNXTensorElementDataType keyType; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetMapKeyType(info, &keyType)); + if (code != ORT_OK) { + return NULL; + } - // Get the ONNXJavaType enum static method - char *javaDataTypeClassName = "ai/onnxruntime/OnnxJavaType"; - jclass onnxJavaTypeClazz = (*jniEnv)->FindClass(jniEnv, javaDataTypeClassName); - jmethodID onnxJavaTypeMapFromONNXTensorType = (*jniEnv)->GetStaticMethodID(jniEnv,onnxJavaTypeClazz, "mapFromOnnxTensorType", "(Lai/onnxruntime/TensorInfo$OnnxTensorType;)Lai/onnxruntime/OnnxJavaType;"); - - // Get the map info class - char *mapInfoClassName = "ai/onnxruntime/MapInfo"; - jclass mapInfoClazz = (*jniEnv)->FindClass(jniEnv, mapInfoClassName); - jmethodID mapInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,mapInfoClazz,"","(ILai/onnxruntime/OnnxJavaType;Lai/onnxruntime/OnnxJavaType;)V"); - - // Extract the key type - ONNXTensorElementDataType keyType; - checkOrtStatus(jniEnv,api,api->GetMapKeyType(info,&keyType)); - - // Convert key type to java - jint onnxTypeKey = convertFromONNXDataFormat(keyType); - jobject onnxTensorTypeJavaKey = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeKey); - jobject onnxJavaTypeKey = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJavaKey); - - // according to include/onnxruntime/core/framework/data_types.h only the following values are supported. - // string, int64, float, double - // So extract the value type, then convert it to a tensor type so we can get it's element type. - OrtTypeInfo* valueTypeInfo; - checkOrtStatus(jniEnv,api,api->GetMapValueType(info,&valueTypeInfo)); - const OrtTensorTypeAndShapeInfo* tensorValueInfo; - checkOrtStatus(jniEnv,api,api->CastTypeInfoToTensorInfo(valueTypeInfo,&tensorValueInfo)); - ONNXTensorElementDataType valueType; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorValueInfo,&valueType)); + // according to include/onnxruntime/core/framework/data_types.h only the following values are supported. + // string, int64, float, double + // So extract the value type, then convert it to a tensor type so we can get it's element type. + OrtTypeInfo* valueTypeInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->GetMapValueType(info, &valueTypeInfo)); + if (code != ORT_OK) { + return NULL; + } + const OrtTensorTypeAndShapeInfo* tensorValueInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->CastTypeInfoToTensorInfo(valueTypeInfo, &tensorValueInfo)); + if (code != ORT_OK) { api->ReleaseTypeInfo(valueTypeInfo); - tensorValueInfo = NULL; - valueTypeInfo = NULL; + return NULL; + } + ONNXTensorElementDataType valueType = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + code = checkOrtStatus(jniEnv, api, api->GetTensorElementType(tensorValueInfo, &valueType)); + api->ReleaseTypeInfo(valueTypeInfo); + tensorValueInfo = NULL; + valueTypeInfo = NULL; + if (code != ORT_OK) { + return NULL; + } - // Convert value type to java - jint onnxTypeValue = convertFromONNXDataFormat(valueType); - jobject onnxTensorTypeJavaValue = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeValue); - jobject onnxJavaTypeValue = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJavaValue); + // Convert key type to java + jint onnxTypeKey = convertFromONNXDataFormat(keyType); + // Convert value type to java + jint onnxTypeValue = convertFromONNXDataFormat(valueType); - // Construct map info - jobject mapInfo = (*jniEnv)->NewObject(jniEnv,mapInfoClazz,mapInfoConstructor,(jint)-1,onnxJavaTypeKey,onnxJavaTypeValue); + // Get the map info class + static const char *mapInfoClassName = "ai/onnxruntime/MapInfo"; + jclass mapInfoClazz = (*jniEnv)->FindClass(jniEnv, mapInfoClassName); + jmethodID mapInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, mapInfoClazz, "", "(III)V"); - return mapInfo; -} + // Construct map info + jobject mapInfo = (*jniEnv)->NewObject(jniEnv, mapInfoClazz, mapInfoConstructor, (jint)-1, onnxTypeKey, onnxTypeValue); -jobject createEmptyMapInfo(JNIEnv *jniEnv) { - // Create the ONNXJavaType enum - char *onnxJavaTypeClassName = "ai/onnxruntime/OnnxJavaType"; - jclass clazz = (*jniEnv)->FindClass(jniEnv, onnxJavaTypeClassName); - jmethodID onnxJavaTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,clazz, "mapFromInt", "(I)Lai/onnxruntime/OnnxJavaType;"); - jobject unknownType = (*jniEnv)->CallStaticObjectMethod(jniEnv,clazz,onnxJavaTypeMapFromInt,0); - - char *mapInfoClassName = "ai/onnxruntime/MapInfo"; - clazz = (*jniEnv)->FindClass(jniEnv, mapInfoClassName); - jmethodID mapInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,clazz,"","(Lai/onnxruntime/OnnxJavaType;Lai/onnxruntime/OnnxJavaType;)V"); - jobject mapInfo = (*jniEnv)->NewObject(jniEnv,clazz,mapInfoConstructor,unknownType,unknownType); - - return mapInfo; + return mapInfo; } jobject convertToSequenceInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtSequenceTypeInfo * info) { - // Get the sequence info class - char *sequenceInfoClassName = "ai/onnxruntime/SequenceInfo"; - jclass sequenceInfoClazz = (*jniEnv)->FindClass(jniEnv, sequenceInfoClassName); + // Get the sequence info class + static const char *sequenceInfoClassName = "ai/onnxruntime/SequenceInfo"; + jclass sequenceInfoClazz = (*jniEnv)->FindClass(jniEnv, sequenceInfoClassName); + jobject sequenceInfo = NULL; - // according to include/onnxruntime/core/framework/data_types.h the following values are supported. - // tensor types, map and map - OrtTypeInfo* elementTypeInfo; - checkOrtStatus(jniEnv,api,api->GetSequenceElementType(info,&elementTypeInfo)); - ONNXType type; - checkOrtStatus(jniEnv,api,api->GetOnnxTypeFromTypeInfo(elementTypeInfo,&type)); + // according to include/onnxruntime/core/framework/data_types.h the following values are supported. + // tensor types, map and map + OrtTypeInfo* elementTypeInfo = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetSequenceElementType(info, &elementTypeInfo)); + if (code != ORT_OK) { + return NULL; + } + ONNXType type = ONNX_TYPE_UNKNOWN; + code = checkOrtStatus(jniEnv, api, api->GetOnnxTypeFromTypeInfo(elementTypeInfo, &type)); + if (code != ORT_OK) { + goto sequence_cleanup; + } - jobject sequenceInfo; + switch (type) { + case ONNX_TYPE_TENSOR: { + // Figure out element type + const OrtTensorTypeAndShapeInfo* elementTensorInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->CastTypeInfoToTensorInfo(elementTypeInfo, &elementTensorInfo)); + if (code != ORT_OK) { + goto sequence_cleanup; + } + ONNXTensorElementDataType element = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + code = checkOrtStatus(jniEnv, api, api->GetTensorElementType(elementTensorInfo, &element)); + if (code != ORT_OK) { + goto sequence_cleanup; + } - switch (type) { - case ONNX_TYPE_TENSOR: { - // Figure out element type - const OrtTensorTypeAndShapeInfo* elementTensorInfo; - checkOrtStatus(jniEnv,api,api->CastTypeInfoToTensorInfo(elementTypeInfo,&elementTensorInfo)); - ONNXTensorElementDataType element; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(elementTensorInfo,&element)); + // Convert element type into ONNXTensorType + jint onnxTypeInt = convertFromONNXDataFormat(element); - // Convert element type into ONNXTensorType - jint onnxTypeInt = convertFromONNXDataFormat(element); - // Get the ONNXTensorType enum static method - char *onnxTensorTypeClassName = "ai/onnxruntime/TensorInfo$OnnxTensorType"; - jclass onnxTensorTypeClazz = (*jniEnv)->FindClass(jniEnv, onnxTensorTypeClassName); - jmethodID onnxTensorTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,onnxTensorTypeClazz, "mapFromInt", "(I)Lai/onnxruntime/TensorInfo$OnnxTensorType;"); - jobject onnxTensorTypeJava = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeInt); - - // Get the ONNXJavaType enum static method - char *javaDataTypeClassName = "ai/onnxruntime/OnnxJavaType"; - jclass onnxJavaTypeClazz = (*jniEnv)->FindClass(jniEnv, javaDataTypeClassName); - jmethodID onnxJavaTypeMapFromONNXTensorType = (*jniEnv)->GetStaticMethodID(jniEnv,onnxJavaTypeClazz, "mapFromOnnxTensorType", "(Lai/onnxruntime/TensorInfo$OnnxTensorType;)Lai/onnxruntime/OnnxJavaType;"); - jobject onnxJavaType = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJava); - - // Construct sequence info - jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,sequenceInfoClazz,"","(ILai/onnxruntime/OnnxJavaType;)V"); - sequenceInfo = (*jniEnv)->NewObject(jniEnv,sequenceInfoClazz,sequenceInfoConstructor,(jint)-1,onnxJavaType); - break; - } - case ONNX_TYPE_MAP: { - // Extract the map info - const OrtMapTypeInfo* mapInfo; - checkOrtStatus(jniEnv,api,api->CastTypeInfoToMapTypeInfo(elementTypeInfo,&mapInfo)); - - // Convert it using the existing convert function - jobject javaMapInfo = convertToMapInfo(jniEnv,api,mapInfo); - - // Construct sequence info - jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,sequenceInfoClazz,"","(ILai/onnxruntime/MapInfo;)V"); - sequenceInfo = (*jniEnv)->NewObject(jniEnv,sequenceInfoClazz,sequenceInfoConstructor,(jint)-1,javaMapInfo); - break; - } - default: { - sequenceInfo = createEmptySequenceInfo(jniEnv); - throwOrtException(jniEnv,convertErrorCode(ORT_INVALID_ARGUMENT),"Invalid element type found in sequence"); - break; - } + // Construct sequence info + jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, sequenceInfoClazz, "", "(II)V"); + sequenceInfo = (*jniEnv)->NewObject(jniEnv, sequenceInfoClazz, sequenceInfoConstructor, (jint)-1, onnxTypeInt); + break; } - api->ReleaseTypeInfo(elementTypeInfo); - elementTypeInfo = NULL; + case ONNX_TYPE_MAP: { + // Extract the map info + const OrtMapTypeInfo* mapInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->CastTypeInfoToMapTypeInfo(elementTypeInfo, &mapInfo)); + if (code != ORT_OK) { + goto sequence_cleanup; + } - return sequenceInfo; + // Convert it using the existing convert function + jobject javaMapInfo = convertToMapInfo(jniEnv, api, mapInfo); + + // Construct sequence info + jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, sequenceInfoClazz, "", "(ILai/onnxruntime/MapInfo;)V"); + sequenceInfo = (*jniEnv)->NewObject(jniEnv, sequenceInfoClazz, sequenceInfoConstructor, (jint)-1, javaMapInfo); + break; + } + default: { + throwOrtException(jniEnv, convertErrorCode(ORT_INVALID_ARGUMENT), "Invalid element type found in sequence"); + break; + } + } + +sequence_cleanup: + api->ReleaseTypeInfo(elementTypeInfo); + elementTypeInfo = NULL; + + return sequenceInfo; } -jobject createEmptySequenceInfo(JNIEnv *jniEnv) { - // Create the ONNXJavaType enum - char *onnxJavaTypeClassName = "ai/onnxruntime/OnnxJavaType"; - jclass clazz = (*jniEnv)->FindClass(jniEnv, onnxJavaTypeClassName); - jmethodID onnxJavaTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,clazz, "mapFromInt", "(I)Lai/onnxruntime/OnnxJavaType;"); - jobject unknownType = (*jniEnv)->CallStaticObjectMethod(jniEnv,clazz,onnxJavaTypeMapFromInt,0); - - char *sequenceInfoClassName = "ai/onnxruntime/SequenceInfo"; - clazz = (*jniEnv)->FindClass(jniEnv, sequenceInfoClassName); - jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,clazz,"","(ILai/onnxruntime/OnnxJavaType;)V"); - jobject sequenceInfo = (*jniEnv)->NewObject(jniEnv,clazz,sequenceInfoConstructor,-1,unknownType); - - return sequenceInfo; -} - -size_t copyJavaToPrimitiveArray(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, jarray input) { - uint32_t inputLength = (*jniEnv)->GetArrayLength(jniEnv,input); - size_t consumedSize = inputLength * onnxTypeSize(onnxType); +int64_t copyJavaToPrimitiveArray(JNIEnv* jniEnv, ONNXTensorElementDataType onnxType, jarray inputArray, uint8_t* outputTensor) { + int32_t inputLength = (*jniEnv)->GetArrayLength(jniEnv, inputArray); + int64_t consumedSize = inputLength * onnxTypeSize(onnxType); switch (onnxType) { case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: // maps to c type uint8_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: { // maps to c type int8_t - jbyteArray typedArr = (jbyteArray) input; - (*jniEnv)->GetByteArrayRegion(jniEnv, typedArr, 0, inputLength, (jbyte * ) tensor); + jbyteArray typedArr = (jbyteArray)inputArray; + (*jniEnv)->GetByteArrayRegion(jniEnv, typedArr, 0, inputLength, (jbyte * )outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: // maps to c type uint16_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: { // maps to c type int16_t - jshortArray typedArr = (jshortArray) input; - (*jniEnv)->GetShortArrayRegion(jniEnv, typedArr, 0, inputLength, (jshort * ) tensor); + jshortArray typedArr = (jshortArray)inputArray; + (*jniEnv)->GetShortArrayRegion(jniEnv, typedArr, 0, inputLength, (jshort * )outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: // maps to c type uint32_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: { // maps to c type int32_t - jintArray typedArr = (jintArray) input; - (*jniEnv)->GetIntArrayRegion(jniEnv, typedArr, 0, inputLength, (jint * ) tensor); + jintArray typedArr = (jintArray)inputArray; + (*jniEnv)->GetIntArrayRegion(jniEnv, typedArr, 0, inputLength, (jint * )outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: // maps to c type uint64_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: { // maps to c type int64_t - jlongArray typedArr = (jlongArray) input; - (*jniEnv)->GetLongArrayRegion(jniEnv, typedArr, 0, inputLength, (jlong * ) tensor); + jlongArray typedArr = (jlongArray)inputArray; + (*jniEnv)->GetLongArrayRegion(jniEnv, typedArr, 0, inputLength, (jlong * )outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: { throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "16-bit float not supported."); - return 0; + return -1; /* float *floatArr = malloc(sizeof(float) * inputLength); - uint16_t *halfArr = (uint16_t *) tensor; + uint16_t *halfArr = (uint16_t *) outputTensor; for (uint32_t i = 0; i < inputLength; i++) { floatArr[i] = convertHalfToFloat(halfArr[i]); } - jfloatArray typedArr = (jfloatArray) input; + jfloatArray typedArr = (jfloatArray) inputArray; (*jniEnv)->GetFloatArrayRegion(jniEnv, typedArr, 0, inputLength, floatArr); free(floatArr); return consumedSize; */ } case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: { // maps to c type float - jfloatArray typedArr = (jfloatArray) input; - (*jniEnv)->GetFloatArrayRegion(jniEnv, typedArr, 0, inputLength, (jfloat * ) tensor); + jfloatArray typedArr = (jfloatArray)inputArray; + (*jniEnv)->GetFloatArrayRegion(jniEnv, typedArr, 0, inputLength, (jfloat * )outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: { // maps to c type double - jdoubleArray typedArr = (jdoubleArray) input; - (*jniEnv)->GetDoubleArrayRegion(jniEnv, typedArr, 0, inputLength, (jdouble * ) tensor); + jdoubleArray typedArr = (jdoubleArray)inputArray; + (*jniEnv)->GetDoubleArrayRegion(jniEnv, typedArr, 0, inputLength, (jdouble * )outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: { // maps to c++ type std::string throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "String is not supported."); - return 0; + return -1; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: { - jbooleanArray typedArr = (jbooleanArray) input; - (*jniEnv)->GetBooleanArrayRegion(jniEnv, typedArr, 0, inputLength, (jboolean *) tensor); + jbooleanArray typedArr = (jbooleanArray)inputArray; + (*jniEnv)->GetBooleanArrayRegion(jniEnv, typedArr, 0, inputLength, (jboolean *)outputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: // complex with float32 real and imaginary components @@ -503,524 +486,566 @@ size_t copyJavaToPrimitiveArray(JNIEnv *jniEnv, ONNXTensorElementDataType onnxTy case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: // Non-IEEE floating-point format based on IEEE754 single-precision case ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED: default: { - throwOrtException(jniEnv, convertErrorCode(ORT_INVALID_ARGUMENT), "Invalid tensor element type."); - return 0; + throwOrtException(jniEnv, convertErrorCode(ORT_INVALID_ARGUMENT), "Invalid outputTensor element type."); + return -1; } } } -size_t copyJavaToTensor(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, size_t tensorSize, - size_t dimensionsRemaining, jarray input) { +int64_t copyJavaToTensor(JNIEnv* jniEnv, ONNXTensorElementDataType onnxType, size_t tensorSize, size_t dimensionsRemaining, jarray inputArray, uint8_t* outputTensor) { if (dimensionsRemaining == 1) { // write out 1d array of the respective primitive type - return copyJavaToPrimitiveArray(jniEnv,onnxType,tensor,input); + return copyJavaToPrimitiveArray(jniEnv, onnxType, inputArray, outputTensor); } else { // recurse through the dimensions // Java arrays are objects until the final dimension - jobjectArray inputObjArr = (jobjectArray) input; - uint32_t dimLength = (*jniEnv)->GetArrayLength(jniEnv,inputObjArr); - size_t sizeConsumed = 0; - for (uint32_t i = 0; i < dimLength; i++) { - jarray childArr = (jarray) (*jniEnv)->GetObjectArrayElement(jniEnv,inputObjArr,i); - sizeConsumed += copyJavaToTensor(jniEnv, onnxType, tensor + sizeConsumed, tensorSize - sizeConsumed, dimensionsRemaining - 1, childArr); + jobjectArray inputObjArr = (jobjectArray)inputArray; + int32_t dimLength = (*jniEnv)->GetArrayLength(jniEnv, inputObjArr); + int64_t sizeConsumed = 0; + for (int32_t i = 0; i < dimLength; i++) { + jarray childArr = (jarray) (*jniEnv)->GetObjectArrayElement(jniEnv, inputObjArr, i); + int64_t consumed = copyJavaToTensor(jniEnv, onnxType, tensorSize - sizeConsumed, dimensionsRemaining - 1, childArr, outputTensor + sizeConsumed); + sizeConsumed += consumed; // Cleanup reference to childArr so it doesn't prevent GC. - (*jniEnv)->DeleteLocalRef(jniEnv,childArr); + (*jniEnv)->DeleteLocalRef(jniEnv, childArr); + // If we failed to copy an array then break and return. + if (consumed == -1) { + return -1; + } } return sizeConsumed; } } -size_t copyPrimitiveArrayToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, jarray output) { - uint32_t outputLength = (*jniEnv)->GetArrayLength(jniEnv,output); +int64_t copyPrimitiveArrayToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, const uint8_t* inputTensor, jarray outputArray) { + int32_t outputLength = (*jniEnv)->GetArrayLength(jniEnv, outputArray); if (outputLength == 0) return 0; - size_t consumedSize = outputLength * onnxTypeSize(onnxType); + int64_t consumedSize = outputLength * onnxTypeSize(onnxType); switch (onnxType) { case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: // maps to c type uint8_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: { // maps to c type int8_t - jbyteArray typedArr = (jbyteArray) output; - (*jniEnv)->SetByteArrayRegion(jniEnv, typedArr, 0, outputLength, (jbyte * ) tensor); + jbyteArray typedArr = (jbyteArray)outputArray; + (*jniEnv)->SetByteArrayRegion(jniEnv, typedArr, 0, outputLength, (jbyte * )inputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: // maps to c type uint16_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: { // maps to c type int16_t - jshortArray typedArr = (jshortArray) output; - (*jniEnv)->SetShortArrayRegion(jniEnv, typedArr, 0, outputLength, (jshort * ) tensor); + jshortArray typedArr = (jshortArray)outputArray; + (*jniEnv)->SetShortArrayRegion(jniEnv, typedArr, 0, outputLength, (jshort * )inputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: // maps to c type uint32_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: { // maps to c type int32_t - jintArray typedArr = (jintArray) output; - (*jniEnv)->SetIntArrayRegion(jniEnv, typedArr, 0, outputLength, (jint * ) tensor); + jintArray typedArr = (jintArray)outputArray; + (*jniEnv)->SetIntArrayRegion(jniEnv, typedArr, 0, outputLength, (jint * )inputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: // maps to c type uint64_t case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: { // maps to c type int64_t - jlongArray typedArr = (jlongArray) output; - (*jniEnv)->SetLongArrayRegion(jniEnv, typedArr, 0, outputLength, (jlong * ) tensor); + jlongArray typedArr = (jlongArray)outputArray; + (*jniEnv)->SetLongArrayRegion(jniEnv, typedArr, 0, outputLength, (jlong * )inputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: { // stored as a uint16_t jfloat *floatArr = malloc(sizeof(jfloat) * outputLength); - if(floatArr == NULL) { + if (floatArr == NULL) { throwOrtException(jniEnv, 1, "Not enough memory"); - return 0; + return -1; } - uint16_t *halfArr = (uint16_t *) tensor; - for (uint32_t i = 0; i < outputLength; i++) { + uint16_t *halfArr = (uint16_t *)inputTensor; + for (int32_t i = 0; i < outputLength; i++) { floatArr[i] = convertHalfToFloat(halfArr[i]); } - jfloatArray typedArr = (jfloatArray) output; + jfloatArray typedArr = (jfloatArray)outputArray; (*jniEnv)->SetFloatArrayRegion(jniEnv, typedArr, 0, outputLength, floatArr); free(floatArr); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: { // maps to c type float - jfloatArray typedArr = (jfloatArray) output; - (*jniEnv)->SetFloatArrayRegion(jniEnv, typedArr, 0, outputLength, (jfloat * ) tensor); + jfloatArray typedArr = (jfloatArray)outputArray; + (*jniEnv)->SetFloatArrayRegion(jniEnv, typedArr, 0, outputLength, (jfloat * )inputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: { // maps to c type double - jdoubleArray typedArr = (jdoubleArray) output; - (*jniEnv)->SetDoubleArrayRegion(jniEnv, typedArr, 0, outputLength, (jdouble * ) tensor); + jdoubleArray typedArr = (jdoubleArray)outputArray; + (*jniEnv)->SetDoubleArrayRegion(jniEnv, typedArr, 0, outputLength, (jdouble * )inputTensor); return consumedSize; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING: { // maps to c++ type std::string // Shouldn't reach here, as it's caught by a different codepath in the initial OnnxTensor.getArray call. throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "String is not supported by this codepath, please raise a Github issue as it should not reach here."); - return 0; + return -1; } case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: { - jbooleanArray typedArr = (jbooleanArray) output; - (*jniEnv)->SetBooleanArrayRegion(jniEnv, typedArr, 0, outputLength, (jboolean *) tensor); + jbooleanArray typedArr = (jbooleanArray)outputArray; + (*jniEnv)->SetBooleanArrayRegion(jniEnv, typedArr, 0, outputLength, (jboolean *)inputTensor); return consumedSize; } - case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: // complex with float32 real and imaginary components - case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: // complex with float64 real and imaginary components - case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: // Non-IEEE floating-point format based on IEEE754 single-precision + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64: { + // complex with float32 real and imaginary components + throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "Invalid inputTensor element type ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64."); + return -1; + } + case ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128: { + // complex with float64 real and imaginary components + throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "Invalid inputTensor element type ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128."); + return -1; + } + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16: { + // Non-IEEE floating-point format based on IEEE754 single-precision + throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "Invalid inputTensor element type ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16."); + return -1; + } case ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED: default: { - throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "Invalid tensor element type."); - return 0; + throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "Invalid inputTensor element type ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED."); + return -1; } } } -size_t copyTensorToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, size_t tensorSize, - size_t dimensionsRemaining, jarray output) { - if (dimensionsRemaining == 1) { - // write out 1d array of the respective primitive type - return copyPrimitiveArrayToJava(jniEnv,onnxType,tensor,output); - } else { - // recurse through the dimensions - // Java arrays are objects until the final dimension - jobjectArray outputObjArr = (jobjectArray) output; - uint32_t dimLength = (*jniEnv)->GetArrayLength(jniEnv,outputObjArr); - size_t sizeConsumed = 0; - for (uint32_t i = 0; i < dimLength; i++) { - jarray childArr = (jarray) (*jniEnv)->GetObjectArrayElement(jniEnv,outputObjArr,i); - sizeConsumed += copyTensorToJava(jniEnv, onnxType, tensor + sizeConsumed, tensorSize - sizeConsumed, dimensionsRemaining - 1, childArr); - // Cleanup reference to childArr so it doesn't prevent GC. - (*jniEnv)->DeleteLocalRef(jniEnv,childArr); - } - return sizeConsumed; +int64_t copyTensorToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, const uint8_t* inputTensor, size_t tensorSize, + size_t dimensionsRemaining, jarray outputArray) { + if (dimensionsRemaining == 1) { + // write out 1d array of the respective primitive type + return copyPrimitiveArrayToJava(jniEnv, onnxType, inputTensor, outputArray); + } else { + // recurse through the dimensions + // Java arrays are objects until the final dimension + jobjectArray outputObjArr = (jobjectArray)outputArray; + int32_t dimLength = (*jniEnv)->GetArrayLength(jniEnv, outputObjArr); + int64_t sizeConsumed = 0; + for (int32_t i = 0; i < dimLength; i++) { + jarray childArr = (jarray) (*jniEnv)->GetObjectArrayElement(jniEnv, outputObjArr, i); + int64_t consumed = copyTensorToJava(jniEnv, onnxType, inputTensor + sizeConsumed, tensorSize - sizeConsumed, dimensionsRemaining - 1, childArr); + sizeConsumed += consumed; + // Cleanup reference to childArr so it doesn't prevent GC. + (*jniEnv)->DeleteLocalRef(jniEnv, childArr); + // If we failed to copy an array then break and return. + if (consumed == -1) { + return -1; + } } + return sizeConsumed; + } } -jobject createStringFromStringTensor(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor) { - // Get the buffer size needed - size_t totalStringLength; - checkOrtStatus(jniEnv,api,api->GetStringTensorDataLength(tensor,&totalStringLength)); +jobject createStringFromStringTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor) { + jobject tempString = NULL; + // Get the buffer size needed + size_t totalStringLength = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetStringTensorDataLength(tensor, &totalStringLength)); + if (code != ORT_OK) { + return NULL; + } - // Create the character and offset buffers - char * characterBuffer; - checkOrtStatus(jniEnv,api,api->AllocatorAlloc(allocator,sizeof(char)*(totalStringLength+1),(void**)&characterBuffer)); - size_t * offsets; - checkOrtStatus(jniEnv,api,api->AllocatorAlloc(allocator,sizeof(size_t),(void**)&offsets)); + // Create the character and offset buffers, character is one larger to allow zero termination. + char * characterBuffer = malloc(sizeof(char)*(totalStringLength+1)); + if (characterBuffer == NULL) { + throwOrtException(jniEnv, 1, "OOM error"); + } else { + size_t * offsets = malloc(sizeof(size_t)); + if (offsets != NULL) { + // Get a view on the String data + code = checkOrtStatus(jniEnv, api, api->GetStringTensorContent(tensor, characterBuffer, totalStringLength, offsets, 1)); - // Get a view on the String data - checkOrtStatus(jniEnv,api,api->GetStringTensorContent(tensor,characterBuffer,totalStringLength,offsets,1)); + if (code == ORT_OK) { + size_t curSize = (offsets[0]) + 1; + characterBuffer[curSize-1] = '\0'; + tempString = (*jniEnv)->NewStringUTF(jniEnv, characterBuffer); + } - size_t curSize = (offsets[0]) + 1; - characterBuffer[curSize-1] = '\0'; - jobject tempString = (*jniEnv)->NewStringUTF(jniEnv,characterBuffer); + free((void*)characterBuffer); + free((void*)offsets); + } + } - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,characterBuffer)); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,offsets)); - - return tempString; + return tempString; } -void copyStringTensorToArray(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor, size_t length, jobjectArray outputArray) { - // Get the buffer size needed - size_t totalStringLength; - checkOrtStatus(jniEnv,api,api->GetStringTensorDataLength(tensor,&totalStringLength)); +OrtErrorCode copyStringTensorToArray(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor, size_t length, jobjectArray outputArray) { + char * tempBuffer = NULL; + // Get the buffer size needed + size_t totalStringLength = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetStringTensorDataLength(tensor, &totalStringLength)); + if (code != ORT_OK) { + return code; + } - // Create the character and offset buffers - char * characterBuffer; - checkOrtStatus(jniEnv,api,api->AllocatorAlloc(allocator,sizeof(char)*(totalStringLength+length),(void**)&characterBuffer)); - // length + 1 as we need to write out the final offset - size_t * offsets; - checkOrtStatus(jniEnv,api,api->AllocatorAlloc(allocator,sizeof(size_t)*(length+1),(void**)&offsets)); - - // Get a view on the String data - checkOrtStatus(jniEnv,api,api->GetStringTensorContent(tensor,characterBuffer,totalStringLength,offsets,length)); + // Create the character and offset buffers + char * characterBuffer = malloc(sizeof(char)*(totalStringLength+length)); + if (characterBuffer == NULL) { + throwOrtException(jniEnv, 1, "Not enough memory"); + return ORT_FAIL; + } + // length + 1 as we need to write out the final offset + size_t * offsets = malloc(sizeof(size_t)*(length+1)); + if (offsets == NULL) { + free((void*)characterBuffer); + throwOrtException(jniEnv, 1, "Not enough memory"); + return ORT_FAIL; + } + // Get a view on the String data + code = checkOrtStatus(jniEnv, api, api->GetStringTensorContent(tensor, characterBuffer, totalStringLength, offsets, length)); + if (code == ORT_OK) { // Get the final offset, write to the end of the array. - checkOrtStatus(jniEnv,api,api->GetStringTensorDataLength(tensor,offsets+length)); - - char * tempBuffer = NULL; - size_t bufferSize = 0; - for (size_t i = 0; i < length; i++) { + code = checkOrtStatus(jniEnv, api, api->GetStringTensorDataLength(tensor, offsets+length)); + if (code == ORT_OK) { + size_t bufferSize = 0; + for (size_t i = 0; i < length; i++) { size_t curSize = (offsets[i+1] - offsets[i]) + 1; if (curSize > bufferSize) { - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,tempBuffer)); - checkOrtStatus(jniEnv,api,api->AllocatorAlloc(allocator,curSize,(void**)&tempBuffer)); - bufferSize = curSize; - } - if (tempBuffer == NULL) { + if (tempBuffer != NULL) { + free((void*)tempBuffer); + } + tempBuffer = malloc(sizeof(char) * curSize); + if (tempBuffer == NULL) { throwOrtException(jniEnv, 1, "Not enough memory"); - return; + goto string_tensor_cleanup; + } + bufferSize = curSize; } memcpy(tempBuffer,characterBuffer+offsets[i],curSize); tempBuffer[curSize-1] = '\0'; jobject tempString = (*jniEnv)->NewStringUTF(jniEnv,tempBuffer); (*jniEnv)->SetObjectArrayElement(jniEnv,outputArray,safecast_size_t_to_jsize(i),tempString); + } } + } - if (tempBuffer != NULL) { - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,tempBuffer)); - } - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,characterBuffer)); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,offsets)); +string_tensor_cleanup: + if (tempBuffer != NULL) { + free((void*)tempBuffer); + } + free((void*)offsets); + free((void*)characterBuffer); + return code; } -jobjectArray createStringArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor) { +jobjectArray createStringArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor) { // Extract tensor info - OrtTensorTypeAndShapeInfo* tensorInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor,&tensorInfo)); + OrtTensorTypeAndShapeInfo* tensorInfo = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetTensorTypeAndShape(tensor, &tensorInfo)); + if (code != ORT_OK) { + return NULL; + } // Get the element count of this tensor - size_t length; - checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo,&length)); + size_t length = 0; + code = checkOrtStatus(jniEnv, api, api->GetTensorShapeElementCount(tensorInfo, &length)); api->ReleaseTensorTypeAndShapeInfo(tensorInfo); + if (code != ORT_OK) { + return NULL; + } // Create the java array - jclass stringClazz = (*jniEnv)->FindClass(jniEnv,"java/lang/String"); - jobjectArray outputArray = (*jniEnv)->NewObjectArray(jniEnv,safecast_size_t_to_jsize(length),stringClazz, NULL); + jclass stringClazz = (*jniEnv)->FindClass(jniEnv, "java/lang/String"); + jobjectArray outputArray = (*jniEnv)->NewObjectArray(jniEnv, safecast_size_t_to_jsize(length), stringClazz, NULL); - copyStringTensorToArray(jniEnv, api, allocator, tensor, length, outputArray); + code = copyStringTensorToArray(jniEnv, api, tensor, length, outputArray); + if (code != ORT_OK) { + outputArray = NULL; + } return outputArray; } jlongArray createLongArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor) { - // Extract tensor type - OrtTensorTypeAndShapeInfo* tensorInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor,&tensorInfo)); - ONNXTensorElementDataType value; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorInfo,&value)); - - // Get the element count of this tensor - size_t length; - checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo,&length)); + jlongArray outputArray = NULL; + // Extract tensor type + OrtTensorTypeAndShapeInfo* tensorInfo = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor, &tensorInfo)); + if (code == ORT_OK) { + ONNXTensorElementDataType value = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + code = checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorInfo, &value)); + if ((code == ORT_OK) && ((value == ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64) || (value == ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64))) { + // Get the element count of this tensor + size_t length = 0; + code = checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo, &length)); + if (code == ORT_OK) { + // Extract the values + uint8_t* arr = NULL; + code = checkOrtStatus(jniEnv,api,api->GetTensorMutableData(tensor, (void**)&arr)); + if (code == ORT_OK) { + // Create the java array and copy to it. + outputArray = (*jniEnv)->NewLongArray(jniEnv, safecast_size_t_to_jsize(length)); + int64_t consumed = copyPrimitiveArrayToJava(jniEnv, value, arr, outputArray); + if (consumed == -1) { + outputArray = NULL; + } + } + } + } api->ReleaseTensorTypeAndShapeInfo(tensorInfo); - - // Extract the values - uint8_t* arr; - checkOrtStatus(jniEnv,api,api->GetTensorMutableData((OrtValue*)tensor,(void**)&arr)); - - // Create the java array and copy to it. - jlongArray outputArray = (*jniEnv)->NewLongArray(jniEnv,safecast_size_t_to_jsize(length)); - copyPrimitiveArrayToJava(jniEnv, value, arr, outputArray); - return outputArray; + } + return outputArray; } jfloatArray createFloatArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor) { - // Extract tensor type - OrtTensorTypeAndShapeInfo* tensorInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor,&tensorInfo)); - ONNXTensorElementDataType value; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorInfo,&value)); - - // Get the element count of this tensor - size_t length; - checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo,&length)); + jfloatArray outputArray = NULL; + // Extract tensor type + OrtTensorTypeAndShapeInfo* tensorInfo = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor, &tensorInfo)); + if (code == ORT_OK) { + ONNXTensorElementDataType value = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + code = checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorInfo, &value)); + if ((code == ORT_OK) && (value == ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT)) { + // Get the element count of this tensor + size_t length = 0; + code = checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo, &length)); + if (code == ORT_OK) { + // Extract the values + uint8_t* arr = NULL; + code = checkOrtStatus(jniEnv,api,api->GetTensorMutableData(tensor, (void**)&arr)); + if (code == ORT_OK) { + // Create the java array and copy to it. + outputArray = (*jniEnv)->NewFloatArray(jniEnv, safecast_size_t_to_jsize(length)); + int64_t consumed = copyPrimitiveArrayToJava(jniEnv, value, arr, outputArray); + if (consumed == -1) { + outputArray = NULL; + } + } + } + } api->ReleaseTensorTypeAndShapeInfo(tensorInfo); - - // Extract the values - uint8_t* arr; - checkOrtStatus(jniEnv,api,api->GetTensorMutableData((OrtValue*)tensor,(void**)&arr)); - - // Create the java array and copy to it. - jfloatArray outputArray = (*jniEnv)->NewFloatArray(jniEnv,safecast_size_t_to_jsize(length)); - copyPrimitiveArrayToJava(jniEnv, value, arr, outputArray); - return outputArray; + } + return outputArray; } jdoubleArray createDoubleArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor) { - // Extract tensor type - OrtTensorTypeAndShapeInfo* tensorInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor,&tensorInfo)); - ONNXTensorElementDataType value; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorInfo,&value)); - - // Get the element count of this tensor - size_t length; - checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo,&length)); + jdoubleArray outputArray = NULL; + // Extract tensor type + OrtTensorTypeAndShapeInfo* tensorInfo = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor, &tensorInfo)); + if (code == ORT_OK) { + ONNXTensorElementDataType value = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + code = checkOrtStatus(jniEnv,api,api->GetTensorElementType(tensorInfo, &value)); + if ((code == ORT_OK) && (value == ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE)) { + // Get the element count of this tensor + size_t length = 0; + code = checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(tensorInfo, &length)); + if (code == ORT_OK) { + // Extract the values + uint8_t* arr = NULL; + code = checkOrtStatus(jniEnv,api,api->GetTensorMutableData(tensor, (void**)&arr)); + if (code == ORT_OK) { + // Create the java array and copy to it. + outputArray = (*jniEnv)->NewDoubleArray(jniEnv, safecast_size_t_to_jsize(length)); + int64_t consumed = copyPrimitiveArrayToJava(jniEnv, value, arr, outputArray); + if (consumed == -1) { + outputArray = NULL; + } + } + } + } api->ReleaseTensorTypeAndShapeInfo(tensorInfo); - - // Extract the values - uint8_t* arr; - checkOrtStatus(jniEnv,api,api->GetTensorMutableData((OrtValue*)tensor,(void**)&arr)); - - // Create the java array and copy to it. - jdoubleArray outputArray = (*jniEnv)->NewDoubleArray(jniEnv,safecast_size_t_to_jsize(length)); - copyPrimitiveArrayToJava(jniEnv, value, arr, outputArray); - return outputArray; + } + return outputArray; } jobject createJavaTensorFromONNX(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor) { - // Extract the type information - OrtTensorTypeAndShapeInfo* info; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(tensor, &info)); + // Extract the type information + OrtTensorTypeAndShapeInfo* info = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetTensorTypeAndShape(tensor, &info)); + if (code != ORT_OK) { + return NULL; + } - // Construct the TensorInfo object - jobject tensorInfo = convertToTensorInfo(jniEnv, api, info); + // Construct the TensorInfo object + jobject tensorInfo = convertToTensorInfo(jniEnv, api, info); + // Release the info object + api->ReleaseTensorTypeAndShapeInfo(info); + if (tensorInfo == NULL) { + return NULL; + } - // Release the info object - api->ReleaseTensorTypeAndShapeInfo(info); + // Construct the ONNXTensor object + static const char *tensorClassName = "ai/onnxruntime/OnnxTensor"; + jclass clazz = (*jniEnv)->FindClass(jniEnv, tensorClassName); + jmethodID tensorConstructor = (*jniEnv)->GetMethodID(jniEnv,clazz, "", "(JJLai/onnxruntime/TensorInfo;)V"); + jobject javaTensor = (*jniEnv)->NewObject(jniEnv, clazz, tensorConstructor, (jlong) tensor, (jlong) allocator, tensorInfo); - // Construct the ONNXTensor object - char *tensorClassName = "ai/onnxruntime/OnnxTensor"; - jclass clazz = (*jniEnv)->FindClass(jniEnv, tensorClassName); - jmethodID tensorConstructor = (*jniEnv)->GetMethodID(jniEnv,clazz, "", "(JJLai/onnxruntime/TensorInfo;)V"); - jobject javaTensor = (*jniEnv)->NewObject(jniEnv, clazz, tensorConstructor, (jlong) tensor, (jlong) allocator, tensorInfo); - - return javaTensor; + return javaTensor; } jobject createJavaSequenceFromONNX(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* sequence) { - // Setup - // Get the ONNXTensorType enum static method - char *onnxTensorTypeClassName = "ai/onnxruntime/TensorInfo$OnnxTensorType"; - jclass onnxTensorTypeClazz = (*jniEnv)->FindClass(jniEnv, onnxTensorTypeClassName); - jmethodID onnxTensorTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,onnxTensorTypeClazz, "mapFromInt", "(I)Lai/onnxruntime/TensorInfo$OnnxTensorType;"); + // Get the sequence info class + static const char *sequenceInfoClassName = "ai/onnxruntime/SequenceInfo"; + jclass sequenceInfoClazz = (*jniEnv)->FindClass(jniEnv, sequenceInfoClassName); - // Get the ONNXJavaType enum static method - char *javaDataTypeClassName = "ai/onnxruntime/OnnxJavaType"; - jclass onnxJavaTypeClazz = (*jniEnv)->FindClass(jniEnv, javaDataTypeClassName); - jmethodID onnxJavaTypeMapFromONNXTensorType = (*jniEnv)->GetStaticMethodID(jniEnv,onnxJavaTypeClazz, "mapFromOnnxTensorType", "(Lai/onnxruntime/TensorInfo$OnnxTensorType;)Lai/onnxruntime/OnnxJavaType;"); - - // Get the sequence info class - char *sequenceInfoClassName = "ai/onnxruntime/SequenceInfo"; - jclass sequenceInfoClazz = (*jniEnv)->FindClass(jniEnv, sequenceInfoClassName); - - // Get the element count of this sequence - size_t count; - checkOrtStatus(jniEnv,api,api->GetValueCount(sequence,&count)); + // setup return value + jobject sequenceInfo = NULL; + // Get the element count of this sequence + size_t count = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetValueCount(sequence, &count)); + if (code != ORT_OK) { + return NULL; + } else if (count == 0) { + // Construct empty sequence info + jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, sequenceInfoClazz, "", "(II)V"); + sequenceInfo = (*jniEnv)->NewObject(jniEnv, sequenceInfoClazz, sequenceInfoConstructor, 0, + convertFromONNXDataFormat(ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED)); + } else { // Extract the first element - OrtValue* firstElement; - checkOrtStatus(jniEnv,api,api->GetValue(sequence,0,allocator,&firstElement)); - ONNXType elementType; - checkOrtStatus(jniEnv,api,api->GetValueType(firstElement,&elementType)); - jobject sequenceInfo; - switch (elementType) { + OrtValue* firstElement = NULL; + code = checkOrtStatus(jniEnv, api, api->GetValue(sequence, 0, allocator, &firstElement)); + if (code != ORT_OK) { + return NULL; + } + ONNXType elementType = ONNX_TYPE_UNKNOWN; + code = checkOrtStatus(jniEnv, api, api->GetValueType(firstElement, &elementType)); + if (code == ORT_OK) { + switch (elementType) { case ONNX_TYPE_TENSOR: { - // Figure out element type - OrtTensorTypeAndShapeInfo* firstElementInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(firstElement,&firstElementInfo)); - ONNXTensorElementDataType element; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(firstElementInfo,&element)); + // Figure out element type + OrtTensorTypeAndShapeInfo* firstElementInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->GetTensorTypeAndShape(firstElement, &firstElementInfo)); + if (code == ORT_OK) { + ONNXTensorElementDataType element = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + code = checkOrtStatus(jniEnv, api, api->GetTensorElementType(firstElementInfo, &element)); api->ReleaseTensorTypeAndShapeInfo(firstElementInfo); + if (code == ORT_OK) { + // Convert element type into ONNXTensorType + jint onnxTypeInt = convertFromONNXDataFormat(element); - // Convert element type into ONNXTensorType - jint onnxTypeInt = convertFromONNXDataFormat(element); - jobject onnxTensorTypeJava = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeInt); - jobject onnxJavaType = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJava); - - // Construct sequence info - jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,sequenceInfoClazz,"","(ILai/onnxruntime/OnnxJavaType;)V"); - sequenceInfo = (*jniEnv)->NewObject(jniEnv,sequenceInfoClazz,sequenceInfoConstructor,(jint)count,onnxJavaType); - break; + // Construct sequence info + jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, sequenceInfoClazz, "", "(II)V"); + sequenceInfo = (*jniEnv)->NewObject(jniEnv, sequenceInfoClazz, sequenceInfoConstructor, (jint)count, onnxTypeInt); + } + } + break; } case ONNX_TYPE_MAP: { - // Extract key - OrtValue* keys; - checkOrtStatus(jniEnv,api,api->GetValue(firstElement,0,allocator,&keys)); - - // Extract key type - OrtTensorTypeAndShapeInfo* keysInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(keys,&keysInfo)); - ONNXTensorElementDataType key; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(keysInfo,&key)); - - // Get the element count of this map - size_t mapCount; - checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(keysInfo,&mapCount)); - - api->ReleaseTensorTypeAndShapeInfo(keysInfo); - - // Convert key type to java - jint onnxTypeKey = convertFromONNXDataFormat(key); - jobject onnxTensorTypeJavaKey = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeKey); - jobject onnxJavaTypeKey = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJavaKey); - - // Extract value - OrtValue* values; - checkOrtStatus(jniEnv,api,api->GetValue(firstElement,1,allocator,&values)); - - // Extract value type - OrtTensorTypeAndShapeInfo* valuesInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(values,&valuesInfo)); - ONNXTensorElementDataType value; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(valuesInfo,&value)); - api->ReleaseTensorTypeAndShapeInfo(valuesInfo); - - // Convert value type to java - jint onnxTypeValue = convertFromONNXDataFormat(value); - jobject onnxTensorTypeJavaValue = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeValue); - jobject onnxJavaTypeValue = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJavaValue); - - // Get the map info class - char *mapInfoClassName = "ai/onnxruntime/MapInfo"; - jclass mapInfoClazz = (*jniEnv)->FindClass(jniEnv, mapInfoClassName); - // Construct map info - jmethodID mapInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,mapInfoClazz,"","(ILai/onnxruntime/OnnxJavaType;Lai/onnxruntime/OnnxJavaType;)V"); - jobject mapInfo = (*jniEnv)->NewObject(jniEnv,mapInfoClazz,mapInfoConstructor,(jint)mapCount,onnxJavaTypeKey,onnxJavaTypeValue); - - // Free the intermediate tensors. - api->ReleaseValue(keys); - api->ReleaseValue(values); - - // Construct sequence info - jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,sequenceInfoClazz,"","(ILai/onnxruntime/MapInfo;)V"); - sequenceInfo = (*jniEnv)->NewObject(jniEnv,sequenceInfoClazz,sequenceInfoConstructor,(jint)count,mapInfo); - break; + jobject mapInfo = createMapInfoFromValue(jniEnv, api, allocator, firstElement); + if (mapInfo != NULL) { + // Construct sequence info + jmethodID sequenceInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, sequenceInfoClazz, "", "(ILai/onnxruntime/MapInfo;)V"); + sequenceInfo = (*jniEnv)->NewObject(jniEnv, sequenceInfoClazz, sequenceInfoConstructor, (jint)count, mapInfo); + } + break; } default: { - sequenceInfo = createEmptySequenceInfo(jniEnv); - throwOrtException(jniEnv,convertErrorCode(ORT_INVALID_ARGUMENT),"Invalid element type found in sequence"); - break; + throwOrtException(jniEnv, convertErrorCode(ORT_INVALID_ARGUMENT), "Invalid element type found in sequence"); + break; } + } } - // Free the intermediate tensor. api->ReleaseValue(firstElement); + } + + jobject javaSequence = NULL; + if (sequenceInfo != NULL) { // Construct the ONNXSequence object - char *sequenceClassName = "ai/onnxruntime/OnnxSequence"; + static const char *sequenceClassName = "ai/onnxruntime/OnnxSequence"; jclass sequenceClazz = (*jniEnv)->FindClass(jniEnv, sequenceClassName); - jmethodID sequenceConstructor = (*jniEnv)->GetMethodID(jniEnv,sequenceClazz, "", "(JJLai/onnxruntime/SequenceInfo;)V"); - jobject javaSequence = (*jniEnv)->NewObject(jniEnv, sequenceClazz, sequenceConstructor, (jlong)sequence, (jlong)allocator, sequenceInfo); + jmethodID sequenceConstructor = (*jniEnv)->GetMethodID(jniEnv, sequenceClazz, "", "(JJLai/onnxruntime/SequenceInfo;)V"); + javaSequence = (*jniEnv)->NewObject(jniEnv, sequenceClazz, sequenceConstructor, (jlong)sequence, (jlong)allocator, sequenceInfo); + } - return javaSequence; + return javaSequence; } jobject createJavaMapFromONNX(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* map) { - // Setup - // Get the ONNXTensorType enum static method - char *onnxTensorTypeClassName = "ai/onnxruntime/TensorInfo$OnnxTensorType"; - jclass onnxTensorTypeClazz = (*jniEnv)->FindClass(jniEnv, onnxTensorTypeClassName); - jmethodID onnxTensorTypeMapFromInt = (*jniEnv)->GetStaticMethodID(jniEnv,onnxTensorTypeClazz, "mapFromInt", "(I)Lai/onnxruntime/TensorInfo$OnnxTensorType;"); + jobject mapInfo = createMapInfoFromValue(jniEnv, api, allocator, map); + if (mapInfo == NULL) { + return NULL; + } - // Get the ONNXJavaType enum static method - char *javaDataTypeClassName = "ai/onnxruntime/OnnxJavaType"; - jclass onnxJavaTypeClazz = (*jniEnv)->FindClass(jniEnv, javaDataTypeClassName); - jmethodID onnxJavaTypeMapFromONNXTensorType = (*jniEnv)->GetStaticMethodID(jniEnv,onnxJavaTypeClazz, "mapFromOnnxTensorType", "(Lai/onnxruntime/TensorInfo$OnnxTensorType;)Lai/onnxruntime/OnnxJavaType;"); + // Get the map class & constructor + static const char *mapClassName = "ai/onnxruntime/OnnxMap"; + jclass mapClazz = (*jniEnv)->FindClass(jniEnv, mapClassName); + jmethodID mapConstructor = (*jniEnv)->GetMethodID(jniEnv, mapClazz, "", "(JJLai/onnxruntime/MapInfo;)V"); - // Get the map info class - char *mapInfoClassName = "ai/onnxruntime/MapInfo"; - jclass mapInfoClazz = (*jniEnv)->FindClass(jniEnv, mapInfoClassName); + // Construct the ONNXMap object + jobject javaMap = (*jniEnv)->NewObject(jniEnv, mapClazz, mapConstructor, (jlong)map, (jlong) allocator, mapInfo); - // Extract key - OrtValue* keys; - checkOrtStatus(jniEnv,api,api->GetValue(map,0,allocator,&keys)); + return javaMap; +} - // Extract key type - OrtTensorTypeAndShapeInfo* keysInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(keys,&keysInfo)); - ONNXTensorElementDataType key; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(keysInfo,&key)); +jobject createMapInfoFromValue(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator * allocator, const OrtValue * map) { + // Extract key + OrtValue* keys = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetValue(map, 0, allocator, &keys)); + if (code != ORT_OK) { + return NULL; + } - // Get the element count of this map - size_t mapCount; - checkOrtStatus(jniEnv,api,api->GetTensorShapeElementCount(keysInfo,&mapCount)); + JavaTensorTypeShape keyInfo; + code = getTensorTypeShape(jniEnv, &keyInfo, api, keys); + api->ReleaseValue(keys); + if (code != ORT_OK) { + return NULL; + } - api->ReleaseTensorTypeAndShapeInfo(keysInfo); + // Extract value + OrtValue* values = NULL; + code = checkOrtStatus(jniEnv, api, api->GetValue(map, 1, allocator, &values)); + if (code != ORT_OK) { + return NULL; + } - // Convert key type to java - jint onnxTypeKey = convertFromONNXDataFormat(key); - jobject onnxTensorTypeJavaKey = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeKey); - jobject onnxJavaTypeKey = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJavaKey); + JavaTensorTypeShape valueInfo; + code = getTensorTypeShape(jniEnv, &valueInfo, api, values); + api->ReleaseValue(values); + if (code != ORT_OK) { + return NULL; + } - // Extract value - OrtValue* values; - checkOrtStatus(jniEnv,api,api->GetValue(map,1,allocator,&values)); + // Convert key and value type to java + jint onnxTypeKey = convertFromONNXDataFormat(keyInfo.onnxTypeEnum); + jint onnxTypeValue = convertFromONNXDataFormat(valueInfo.onnxTypeEnum); - // Extract value type - OrtTensorTypeAndShapeInfo* valuesInfo; - checkOrtStatus(jniEnv,api,api->GetTensorTypeAndShape(values,&valuesInfo)); - ONNXTensorElementDataType value; - checkOrtStatus(jniEnv,api,api->GetTensorElementType(valuesInfo,&value)); - api->ReleaseTensorTypeAndShapeInfo(valuesInfo); + // Get the map info class & constructor + static const char *mapInfoClassName = "ai/onnxruntime/MapInfo"; + jclass mapInfoClazz = (*jniEnv)->FindClass(jniEnv, mapInfoClassName); + jmethodID mapInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, mapInfoClazz, "", "(III)V"); - // Convert value type to java - jint onnxTypeValue = convertFromONNXDataFormat(value); - jobject onnxTensorTypeJavaValue = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxTensorTypeClazz,onnxTensorTypeMapFromInt,onnxTypeValue); - jobject onnxJavaTypeValue = (*jniEnv)->CallStaticObjectMethod(jniEnv,onnxJavaTypeClazz,onnxJavaTypeMapFromONNXTensorType,onnxTensorTypeJavaValue); - - // Construct map info - jmethodID mapInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,mapInfoClazz,"","(ILai/onnxruntime/OnnxJavaType;Lai/onnxruntime/OnnxJavaType;)V"); - jobject mapInfo = (*jniEnv)->NewObject(jniEnv,mapInfoClazz,mapInfoConstructor,(jint)mapCount,onnxJavaTypeKey,onnxJavaTypeValue); - - // Free the intermediate tensors. - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,keys)); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,values)); - - // Construct the ONNXMap object - char *mapClassName = "ai/onnxruntime/OnnxMap"; - jclass mapClazz = (*jniEnv)->FindClass(jniEnv, mapClassName); - jmethodID mapConstructor = (*jniEnv)->GetMethodID(jniEnv,mapClazz, "", "(JJLai/onnxruntime/MapInfo;)V"); - jobject javaMap = (*jniEnv)->NewObject(jniEnv, mapClazz, mapConstructor, (jlong)map, (jlong) allocator, mapInfo); - - return javaMap; + // Construct map info + jobject mapInfo = (*jniEnv)->NewObject(jniEnv, mapInfoClazz, mapInfoConstructor, (jint)keyInfo.elementCount, onnxTypeKey, onnxTypeValue); + return mapInfo; } jobject convertOrtValueToONNXValue(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* onnxValue) { - // Note this is the ONNXType C enum - ONNXType valueType; - checkOrtStatus(jniEnv,api,api->GetValueType(onnxValue,&valueType)); - switch (valueType) { - case ONNX_TYPE_TENSOR: { - return createJavaTensorFromONNX(jniEnv, api, allocator, onnxValue); - } - case ONNX_TYPE_SEQUENCE: { - return createJavaSequenceFromONNX(jniEnv, api, allocator, onnxValue); - } - case ONNX_TYPE_MAP: { - return createJavaMapFromONNX(jniEnv, api, allocator, onnxValue); - } - case ONNX_TYPE_UNKNOWN: - case ONNX_TYPE_OPAQUE: - case ONNX_TYPE_OPTIONAL: - case ONNX_TYPE_SPARSETENSOR: { - throwOrtException(jniEnv,convertErrorCode(ORT_NOT_IMPLEMENTED),"These types are unsupported - ONNX_TYPE_UNKNOWN, ONNX_TYPE_OPAQUE, ONNX_TYPE_SPARSETENSOR."); - break; - } - } + // Note this is the ONNXType C enum + ONNXType valueType = ONNX_TYPE_UNKNOWN; + OrtErrorCode code = checkOrtStatus(jniEnv,api,api->GetValueType(onnxValue,&valueType)); + if (code != ORT_OK) { return NULL; + } + switch (valueType) { + case ONNX_TYPE_TENSOR: { + return createJavaTensorFromONNX(jniEnv, api, allocator, onnxValue); + } + case ONNX_TYPE_SEQUENCE: { + return createJavaSequenceFromONNX(jniEnv, api, allocator, onnxValue); + } + case ONNX_TYPE_MAP: { + return createJavaMapFromONNX(jniEnv, api, allocator, onnxValue); + } + case ONNX_TYPE_UNKNOWN: + case ONNX_TYPE_OPAQUE: + case ONNX_TYPE_OPTIONAL: + case ONNX_TYPE_SPARSETENSOR: + default: { + throwOrtException(jniEnv, convertErrorCode(ORT_NOT_IMPLEMENTED), "These types are unsupported - ONNX_TYPE_UNKNOWN, ONNX_TYPE_OPAQUE, ONNX_TYPE_SPARSETENSOR."); + return NULL; + } + } } jint throwOrtException(JNIEnv *jniEnv, int messageId, const char *message) { - jstring messageStr = (*jniEnv)->NewStringUTF(jniEnv, message); + jstring messageStr = (*jniEnv)->NewStringUTF(jniEnv, message); - char *className = "ai/onnxruntime/OrtException"; - jclass exClazz = (*jniEnv)->FindClass(jniEnv,className); - jmethodID exConstructor = (*jniEnv)->GetMethodID(jniEnv, exClazz, "", "(ILjava/lang/String;)V"); - jobject javaException = (*jniEnv)->NewObject(jniEnv, exClazz, exConstructor, messageId, messageStr); + static const char *className = "ai/onnxruntime/OrtException"; + jclass exClazz = (*jniEnv)->FindClass(jniEnv, className); + jmethodID exConstructor = (*jniEnv)->GetMethodID(jniEnv, exClazz, "", "(ILjava/lang/String;)V"); + jobject javaException = (*jniEnv)->NewObject(jniEnv, exClazz, exConstructor, messageId, messageStr); - return (*jniEnv)->Throw(jniEnv,javaException); + return (*jniEnv)->Throw(jniEnv, javaException); } jint convertErrorCode(OrtErrorCode code) { diff --git a/java/src/main/native/OrtJniUtil.h b/java/src/main/native/OrtJniUtil.h index 075202333e..616a20503a 100644 --- a/java/src/main/native/OrtJniUtil.h +++ b/java/src/main/native/OrtJniUtil.h @@ -38,29 +38,27 @@ OrtErrorCode getTensorTypeShape(JNIEnv * jniEnv, JavaTensorTypeShape * output, c jfloat convertHalfToFloat(uint16_t half); -jobject convertToValueInfo(JNIEnv *jniEnv, const OrtApi * api, OrtTypeInfo * info); +jobject convertToValueInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtTypeInfo * info); jobject convertToTensorInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtTensorTypeAndShapeInfo * info); jobject convertToMapInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtMapTypeInfo * info); + jobject convertToSequenceInfo(JNIEnv *jniEnv, const OrtApi * api, const OrtSequenceTypeInfo * info); -jobject createEmptyMapInfo(JNIEnv *jniEnv); -jobject createEmptySequenceInfo(JNIEnv *jniEnv); +int64_t copyJavaToPrimitiveArray(JNIEnv* jniEnv, ONNXTensorElementDataType onnxType, jarray inputArray, uint8_t* outputTensor); -size_t copyJavaToPrimitiveArray(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, jarray input); +int64_t copyJavaToTensor(JNIEnv* jniEnv, ONNXTensorElementDataType onnxType, size_t tensorSize, size_t dimensionsRemaining, jarray inputArray, uint8_t* outputTensor); -size_t copyJavaToTensor(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, size_t tensorSize, size_t dimensionsRemaining, jarray input); +int64_t copyPrimitiveArrayToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, const uint8_t* inputTensor, jarray outputArray); -size_t copyPrimitiveArrayToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, jarray output); +int64_t copyTensorToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, const uint8_t* inputTensor, size_t tensorSize, size_t dimensionsRemaining, jarray outputArray); -size_t copyTensorToJava(JNIEnv *jniEnv, ONNXTensorElementDataType onnxType, uint8_t* tensor, size_t tensorSize, size_t dimensionsRemaining, jarray output); +jobject createStringFromStringTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor); -jobject createStringFromStringTensor(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor); +OrtErrorCode copyStringTensorToArray(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor, size_t length, jobjectArray outputArray); -void copyStringTensorToArray(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor, size_t length, jobjectArray outputArray); - -jobjectArray createStringArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* tensor); +jobjectArray createStringArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor); jlongArray createLongArrayFromTensor(JNIEnv *jniEnv, const OrtApi * api, OrtValue* tensor); @@ -74,6 +72,8 @@ jobject createJavaSequenceFromONNX(JNIEnv *jniEnv, const OrtApi * api, OrtAlloca jobject createJavaMapFromONNX(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* map); +jobject createMapInfoFromValue(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator * allocator, const OrtValue * map); + jobject convertOrtValueToONNXValue(JNIEnv *jniEnv, const OrtApi * api, OrtAllocator* allocator, OrtValue* onnxValue); jint throwOrtException(JNIEnv *env, int messageId, const char *message); diff --git a/java/src/main/native/ai_onnxruntime_OnnxMap.c b/java/src/main/native/ai_onnxruntime_OnnxMap.c index dce0677c84..161fa23ae8 100644 --- a/java/src/main/native/ai_onnxruntime_OnnxMap.c +++ b/java/src/main/native/ai_onnxruntime_OnnxMap.c @@ -22,7 +22,7 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OnnxMap_getStringKeys(JNIEnv* OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetValue((OrtValue*)handle, 0, allocator, &keys)); if (code == ORT_OK) { // Convert to Java String array - jobjectArray output = createStringArrayFromTensor(jniEnv, api, allocator, keys); + jobjectArray output = createStringArrayFromTensor(jniEnv, api, keys); api->ReleaseValue(keys); @@ -72,7 +72,7 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OnnxMap_getStringValues(JNIEn OrtErrorCode code = checkOrtStatus(jniEnv, api, api->GetValue((OrtValue*)handle, 1, allocator, &values)); if (code == ORT_OK) { // Convert to Java String array - jobjectArray output = createStringArrayFromTensor(jniEnv, api, allocator, values); + jobjectArray output = createStringArrayFromTensor(jniEnv, api, values); api->ReleaseValue(values); diff --git a/java/src/main/native/ai_onnxruntime_OnnxSequence.c b/java/src/main/native/ai_onnxruntime_OnnxSequence.c index 12d3266f8a..22cfca6315 100644 --- a/java/src/main/native/ai_onnxruntime_OnnxSequence.c +++ b/java/src/main/native/ai_onnxruntime_OnnxSequence.c @@ -28,7 +28,7 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OnnxSequence_getStringKeys(JN if (code == ORT_OK) { // Convert to Java String array - output = createStringArrayFromTensor(jniEnv, api, allocator, keys); + output = createStringArrayFromTensor(jniEnv, api, keys); // Release if valid api->ReleaseValue(element); } @@ -94,7 +94,7 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OnnxSequence_getStringValues( if (code == ORT_OK) { // Convert to Java String array - output = createStringArrayFromTensor(jniEnv, api, allocator, values); + output = createStringArrayFromTensor(jniEnv, api, values); // Release if valid api->ReleaseValue(element); } @@ -230,7 +230,7 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OnnxSequence_getStrings(JNIEn OrtValue* element; code = checkOrtStatus(jniEnv, api, api->GetValue(sequence, (int)i, allocator, &element)); if (code == ORT_OK) { - jobject str = createStringFromStringTensor(jniEnv, api, allocator, element); + jobject str = createStringFromStringTensor(jniEnv, api, element); if (str == NULL) { api->ReleaseValue(element); // bail out as exception has been thrown diff --git a/java/src/main/native/ai_onnxruntime_OnnxTensor.c b/java/src/main/native/ai_onnxruntime_OnnxTensor.c index 06a8ad42e7..1656b4043c 100644 --- a/java/src/main/native/ai_onnxruntime_OnnxTensor.c +++ b/java/src/main/native/ai_onnxruntime_OnnxTensor.c @@ -44,8 +44,8 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OnnxTensor_createTensor // Check if we're copying a scalar or not if (shapeLen == 0) { // Scalars are passed in as a single element array - size_t copied = copyJavaToPrimitiveArray(jniEnv, onnxType, tensorData, dataObj); - failed = copied == 0 ? 1 : failed; + int64_t copied = copyJavaToPrimitiveArray(jniEnv, onnxType, dataObj, tensorData); + failed = copied == -1 ? 1 : failed; } else { // Extract the tensor shape information JavaTensorTypeShape typeShape; @@ -53,9 +53,9 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OnnxTensor_createTensor if (code == ORT_OK) { // Copy the java array into the tensor - size_t copied = copyJavaToTensor(jniEnv, onnxType, tensorData, typeShape.elementCount, - typeShape.dimensions, dataObj); - failed = copied == 0 ? 1 : failed; + int64_t copied = copyJavaToTensor(jniEnv, onnxType, typeShape.elementCount, + typeShape.dimensions, dataObj, tensorData); + failed = copied == -1 ? 1 : failed; } else { failed = 1; } @@ -367,12 +367,11 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OnnxTensor_getLong * Signature: (JJ)Ljava/lang/String; */ JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OnnxTensor_getString - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle, jlong allocatorHandle) { + (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle) { (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. const OrtApi* api = (const OrtApi*) apiHandle; // Extract a String array - if this becomes a performance issue we'll refactor later. - jobjectArray outputArray = createStringArrayFromTensor(jniEnv, api, (OrtAllocator*) allocatorHandle, - (OrtValue*) handle); + jobjectArray outputArray = createStringArrayFromTensor(jniEnv, api, (OrtValue*) handle); if (outputArray != NULL) { // Get reference to the string jobject output = (*jniEnv)->GetObjectArrayElement(jniEnv, outputArray, 0); @@ -410,7 +409,7 @@ JNIEXPORT jboolean JNICALL Java_ai_onnxruntime_OnnxTensor_getBool * Signature: (JJLjava/lang/Object;)V */ JNIEXPORT void JNICALL Java_ai_onnxruntime_OnnxTensor_getArray - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle, jlong allocatorHandle, jobject carrier) { + (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle, jobject carrier) { (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. const OrtApi* api = (const OrtApi*) apiHandle; OrtValue* value = (OrtValue*) handle; @@ -418,7 +417,7 @@ JNIEXPORT void JNICALL Java_ai_onnxruntime_OnnxTensor_getArray OrtErrorCode code = getTensorTypeShape(jniEnv, &typeShape, api, value); if (code == ORT_OK) { if (typeShape.onnxTypeEnum == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) { - copyStringTensorToArray(jniEnv, api, (OrtAllocator*) allocatorHandle, value, typeShape.elementCount, carrier); + copyStringTensorToArray(jniEnv, api, value, typeShape.elementCount, carrier); } else { uint8_t* arr = NULL; code = checkOrtStatus(jniEnv, api, api->GetTensorMutableData(value, (void**)&arr)); diff --git a/java/src/main/native/ai_onnxruntime_OrtSession.c b/java/src/main/native/ai_onnxruntime_OrtSession.c index de24f10f56..07354ffd1e 100644 --- a/java/src/main/native/ai_onnxruntime_OrtSession.c +++ b/java/src/main/native/ai_onnxruntime_OrtSession.c @@ -4,40 +4,47 @@ */ #include #include +#include #include "onnxruntime/core/session/onnxruntime_c_api.h" #include "OrtJniUtil.h" #include "ai_onnxruntime_OrtSession.h" +const char * const ORTJNI_StringClassName = "java/lang/String"; +const char * const ORTJNI_OnnxValueClassName = "ai/onnxruntime/OnnxValue"; +const char * const ORTJNI_NodeInfoClassName = "ai/onnxruntime/NodeInfo"; +const char * const ORTJNI_MetadataClassName = "ai/onnxruntime/OnnxModelMetadata"; + /* * Class: ai_onnxruntime_OrtSession * Method: createSession * Signature: (JJLjava/lang/String;J)J */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_createSession__JJLjava_lang_String_2J - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong envHandle, jstring modelPath, jlong optsHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtSession* session; +JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_createSession__JJLjava_lang_String_2J(JNIEnv* jniEnv, jclass jclazz, jlong apiHandle, jlong envHandle, jstring modelPath, jlong optsHandle) { + (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtSession* session = NULL; #ifdef _WIN32 - const jchar* cPath = (*jniEnv)->GetStringChars(jniEnv, modelPath, NULL); - size_t stringLength = (*jniEnv)->GetStringLength(jniEnv, modelPath); - wchar_t* newString = (wchar_t*)calloc(stringLength + 1, sizeof(wchar_t)); - if(newString == NULL) { - throwOrtException(jniEnv, 1, "Not enough memory"); - return 0; - } - wcsncpy_s(newString, stringLength+1, (const wchar_t*) cPath, stringLength); - checkOrtStatus(jniEnv,api,api->CreateSession((OrtEnv*)envHandle, newString, (OrtSessionOptions*)optsHandle, &session)); - free(newString); - (*jniEnv)->ReleaseStringChars(jniEnv,modelPath,cPath); + const jchar* cPath = (*jniEnv)->GetStringChars(jniEnv, modelPath, NULL); + size_t stringLength = (*jniEnv)->GetStringLength(jniEnv, modelPath); + wchar_t* newString = (wchar_t*)calloc(stringLength + 1, sizeof(wchar_t)); + if (newString == NULL) { + (*jniEnv)->ReleaseStringChars(jniEnv, modelPath, cPath); + throwOrtException(jniEnv, 1, "Not enough memory"); + return 0; + } + wcsncpy_s(newString, stringLength + 1, (const wchar_t*)cPath, stringLength); + checkOrtStatus(jniEnv, api, + api->CreateSession((OrtEnv*)envHandle, newString, (OrtSessionOptions*)optsHandle, &session)); + free(newString); + (*jniEnv)->ReleaseStringChars(jniEnv, modelPath, cPath); #else - const char* cPath = (*jniEnv)->GetStringUTFChars(jniEnv, modelPath, NULL); - checkOrtStatus(jniEnv,api,api->CreateSession((OrtEnv*)envHandle, cPath, (OrtSessionOptions*)optsHandle, &session)); - (*jniEnv)->ReleaseStringUTFChars(jniEnv,modelPath,cPath); + const char* cPath = (*jniEnv)->GetStringUTFChars(jniEnv, modelPath, NULL); + checkOrtStatus(jniEnv, api, api->CreateSession((OrtEnv*)envHandle, cPath, (OrtSessionOptions*)optsHandle, &session)); + (*jniEnv)->ReleaseStringUTFChars(jniEnv, modelPath, cPath); #endif - return (jlong) session; + return (jlong)session; } /* @@ -45,34 +52,39 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_createSession__JJLjava_la * Method: createSession * Signature: (JJ[BJ)J */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_createSession__JJ_3BJ - (JNIEnv * jniEnv, jclass jclazz, jlong apiHandle, jlong envHandle, jbyteArray jModelArray, jlong optsHandle) { - (void) jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtSession* session; +JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_createSession__JJ_3BJ(JNIEnv* jniEnv, jclass jclazz, jlong apiHandle, jlong envHandle, jbyteArray jModelArray, jlong optsHandle) { + (void)jclazz; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtEnv* env = (OrtEnv*)envHandle; + OrtSessionOptions* opts = (OrtSessionOptions*)optsHandle; + OrtSession* session = NULL; - // Get a reference to the byte array elements - jbyte* modelArr = (*jniEnv)->GetByteArrayElements(jniEnv,jModelArray,NULL); - size_t modelLength = (*jniEnv)->GetArrayLength(jniEnv,jModelArray); - checkOrtStatus(jniEnv,api,api->CreateSessionFromArray((OrtEnv*)envHandle, modelArr, modelLength, (OrtSessionOptions*)optsHandle, &session)); - // Release the C array. - (*jniEnv)->ReleaseByteArrayElements(jniEnv,jModelArray,modelArr,JNI_ABORT); - - return (jlong) session; + size_t modelLength = (*jniEnv)->GetArrayLength(jniEnv, jModelArray); + if (modelLength == 0) { + throwOrtException(jniEnv, 2, "Invalid ONNX model, the byte array is zero length."); + return 0; } + // Get a reference to the byte array elements + jbyte* modelArr = (*jniEnv)->GetByteArrayElements(jniEnv, jModelArray, NULL); + checkOrtStatus(jniEnv, api, api->CreateSessionFromArray(env, modelArr, modelLength, opts, &session)); + // Release the C array. + (*jniEnv)->ReleaseByteArrayElements(jniEnv, jModelArray, modelArr, JNI_ABORT); + + return (jlong)session; +} + /* * Class: ai_onnxruntime_OrtSession * Method: getNumInputs * Signature: (JJ)J */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getNumInputs - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - size_t numInputs; - checkOrtStatus(jniEnv,api,api->SessionGetInputCount((OrtSession*)handle, &numInputs)); - return numInputs; +JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getNumInputs(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong handle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + size_t numInputs = 0; + checkOrtStatus(jniEnv, api, api->SessionGetInputCount((OrtSession*)handle, &numInputs)); + return numInputs; } /* @@ -80,31 +92,47 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getNumInputs * Method: getInputNames * Signature: (JJJ)[Ljava/lang/String; */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getInputNames - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; +JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getInputNames(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; + OrtSession* session = (OrtSession*)sessionHandle; - // Setup - char *stringClassName = "java/lang/String"; - jclass stringClazz = (*jniEnv)->FindClass(jniEnv, stringClassName); + // Setup + jclass stringClazz = (*jniEnv)->FindClass(jniEnv, ORTJNI_StringClassName); - // Get the number of inputs - size_t numInputs = Java_ai_onnxruntime_OrtSession_getNumInputs(jniEnv, jobj, apiHandle, sessionHandle); + // Get the number of inputs + size_t numInputs = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->SessionGetInputCount(session, &numInputs)); + if (code != ORT_OK) { + return NULL; + } - // Allocate the return array - jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv,safecast_size_t_to_jsize(numInputs),stringClazz,NULL); - for (uint32_t i = 0; i < numInputs; i++) { - // Read out the input name and convert it to a java.lang.String - char* inputName; - checkOrtStatus(jniEnv,api,api->SessionGetInputName((OrtSession*)sessionHandle, i, allocator, &inputName)); - jstring name = (*jniEnv)->NewStringUTF(jniEnv,inputName); - (*jniEnv)->SetObjectArrayElement(jniEnv, array, i, name); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,inputName)); + int32_t numInputsInt = (int32_t) numInputs; + if (numInputs != (size_t) numInputsInt) { + throwOrtException(jniEnv, 1, "Too many inputs, expected less than 2^31"); + } + + // Allocate the return array + jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv, numInputsInt, stringClazz, NULL); + for (int32_t i = 0; i < numInputsInt; i++) { + // Read out the input name and convert it to a java.lang.String + char* inputName = NULL; + code = checkOrtStatus(jniEnv, api, api->SessionGetInputName(session, i, allocator, &inputName)); + if (code != ORT_OK) { + // break out on error, return array and let Java throw the exception. + break; } + jstring name = (*jniEnv)->NewStringUTF(jniEnv, inputName); + (*jniEnv)->SetObjectArrayElement(jniEnv, array, i, name); + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, inputName)); + if (code != ORT_OK) { + // break out on error, return array and let Java throw the exception. + break; + } + } - return array; + return array; } /* @@ -112,13 +140,12 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getInputNames * Method: getNumOutputs * Signature: (JJ)J */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getNumOutputs - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - size_t numOutputs; - checkOrtStatus(jniEnv,api,api->SessionGetOutputCount((OrtSession*)handle, &numOutputs)); - return numOutputs; +JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getNumOutputs(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong handle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + size_t numOutputs = 0; + checkOrtStatus(jniEnv, api, api->SessionGetOutputCount((OrtSession*)handle, &numOutputs)); + return numOutputs; } /* @@ -126,31 +153,47 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getNumOutputs * Method: getOutputNames * Signature: (JJJ)[Ljava/lang/String; */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getOutputNames - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; +JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getOutputNames(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtSession* session = (OrtSession*)sessionHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; - // Setup - char *stringClassName = "java/lang/String"; - jclass stringClazz = (*jniEnv)->FindClass(jniEnv, stringClassName); + // Setup + jclass stringClazz = (*jniEnv)->FindClass(jniEnv, ORTJNI_StringClassName); - // Get the number of outputs - size_t numOutputs = Java_ai_onnxruntime_OrtSession_getNumOutputs(jniEnv, jobj, apiHandle, sessionHandle); + // Get the number of outputs + size_t numOutputs = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->SessionGetOutputCount(session, &numOutputs)); + if (code != ORT_OK) { + return NULL; + } - // Allocate the return array - jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv,safecast_size_t_to_jsize(numOutputs),stringClazz, NULL); - for (uint32_t i = 0; i < numOutputs; i++) { - // Read out the output name and convert it to a java.lang.String - char* outputName; - checkOrtStatus(jniEnv,api,api->SessionGetOutputName((OrtSession*)sessionHandle, i, allocator, &outputName)); - jstring name = (*jniEnv)->NewStringUTF(jniEnv,outputName); - (*jniEnv)->SetObjectArrayElement(jniEnv, array, i, name); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,outputName)); + int32_t numOutputsInt = (int32_t) numOutputs; + if (numOutputs != (size_t) numOutputsInt) { + throwOrtException(jniEnv, 1, "Too many outputs, expected less than 2^31"); + } + + // Allocate the return array + jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv, numOutputsInt, stringClazz, NULL); + for (int32_t i = 0; i < numOutputsInt; i++) { + // Read out the output name and convert it to a java.lang.String + char* outputName = NULL; + code = checkOrtStatus(jniEnv, api, api->SessionGetOutputName(session, i, allocator, &outputName)); + if (code != ORT_OK) { + // break out on error, return array and let Java throw the exception. + break; } + jstring name = (*jniEnv)->NewStringUTF(jniEnv, outputName); + (*jniEnv)->SetObjectArrayElement(jniEnv, array, i, name); + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, outputName)); + if (code != ORT_OK) { + // break out on error, return array and let Java throw the exception. + break; + } + } - return array; + return array; } /* @@ -158,41 +201,57 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getOutputNames * Method: getInputInfo * Signature: (JJJ)[Lai/onnxruntime/NodeInfo; */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getInputInfo - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; +JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getInputInfo(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtSession* session = (OrtSession*)sessionHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; - // Setup - char *nodeInfoClassName = "ai/onnxruntime/NodeInfo"; - jclass nodeInfoClazz = (*jniEnv)->FindClass(jniEnv, nodeInfoClassName); - jmethodID nodeInfoConstructor = (*jniEnv)->GetMethodID(jniEnv,nodeInfoClazz, "", "(Ljava/lang/String;Lai/onnxruntime/ValueInfo;)V"); + // Setup + jclass nodeInfoClazz = (*jniEnv)->FindClass(jniEnv, ORTJNI_NodeInfoClassName); + jmethodID nodeInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, nodeInfoClazz, "", + "(Ljava/lang/String;Lai/onnxruntime/ValueInfo;)V"); - // Get the number of inputs - size_t numInputs = Java_ai_onnxruntime_OrtSession_getNumInputs(jniEnv, jobj, apiHandle, sessionHandle); + // Get the number of inputs + size_t numInputs = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->SessionGetInputCount(session, &numInputs)); + if (code != ORT_OK) { + return NULL; + } - // Allocate the return array - jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv,safecast_size_t_to_jsize(numInputs),nodeInfoClazz, NULL); - for (size_t i = 0; i < numInputs; i++) { - // Read out the input name and convert it to a java.lang.String - char* inputName; - checkOrtStatus(jniEnv,api,api->SessionGetInputName((OrtSession*)sessionHandle, i, allocator, &inputName)); - jstring name = (*jniEnv)->NewStringUTF(jniEnv,inputName); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,inputName)); - - // Create a ValueInfo from the OrtTypeInfo - OrtTypeInfo* typeInfo; - checkOrtStatus(jniEnv,api,api->SessionGetInputTypeInfo((OrtSession*)sessionHandle, i, &typeInfo)); - jobject valueInfoJava = convertToValueInfo(jniEnv,api,typeInfo); - api->ReleaseTypeInfo(typeInfo); - - // Create a NodeInfo and assign into the array - jobject nodeInfo = (*jniEnv)->NewObject(jniEnv, nodeInfoClazz, nodeInfoConstructor, name, valueInfoJava); - (*jniEnv)->SetObjectArrayElement(jniEnv, array,safecast_size_t_to_jsize(i),nodeInfo); + // Allocate the return array + jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv, safecast_size_t_to_jsize(numInputs), nodeInfoClazz, NULL); + for (size_t i = 0; i < numInputs; i++) { + // Read out the input name and convert it to a java.lang.String + char* inputName = NULL; + code = checkOrtStatus(jniEnv, api, api->SessionGetInputName(session, i, allocator, &inputName)); + if (code != ORT_OK) { + break; + } + jstring name = (*jniEnv)->NewStringUTF(jniEnv, inputName); + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, inputName)); + if (code != ORT_OK) { + break; } - return array; + // Create a ValueInfo from the OrtTypeInfo + OrtTypeInfo* typeInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->SessionGetInputTypeInfo(session, i, &typeInfo)); + if (code != ORT_OK) { + break; + } + jobject valueInfoJava = convertToValueInfo(jniEnv, api, typeInfo); + api->ReleaseTypeInfo(typeInfo); + if (valueInfoJava == NULL) { + break; + } + + // Create a NodeInfo and assign into the array + jobject nodeInfo = (*jniEnv)->NewObject(jniEnv, nodeInfoClazz, nodeInfoConstructor, name, valueInfoJava); + (*jniEnv)->SetObjectArrayElement(jniEnv, array, safecast_size_t_to_jsize(i), nodeInfo); + } + + return array; } /* @@ -200,40 +259,57 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getInputInfo * Method: getOutputInfo * Signature: (JJJ)[Lai/onnxruntime/NodeInfo; */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getOutputInfo - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; - // Setup - char *nodeInfoClassName = "ai/onnxruntime/NodeInfo"; - jclass nodeInfoClazz = (*jniEnv)->FindClass(jniEnv, nodeInfoClassName); - jmethodID nodeInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, nodeInfoClazz, "", "(Ljava/lang/String;Lai/onnxruntime/ValueInfo;)V"); +JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getOutputInfo(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtSession* session = (OrtSession*)sessionHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; - // Get the number of outputs - size_t numOutputs = Java_ai_onnxruntime_OrtSession_getNumOutputs(jniEnv, jobj, apiHandle, sessionHandle); + // Setup + jclass nodeInfoClazz = (*jniEnv)->FindClass(jniEnv, ORTJNI_NodeInfoClassName); + jmethodID nodeInfoConstructor = (*jniEnv)->GetMethodID(jniEnv, nodeInfoClazz, "", + "(Ljava/lang/String;Lai/onnxruntime/ValueInfo;)V"); - // Allocate the return array - jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv,safecast_size_t_to_jsize(numOutputs),nodeInfoClazz,NULL); - for (uint32_t i = 0; i < numOutputs; i++) { - // Read out the output name and convert it to a java.lang.String - char* outputName; - checkOrtStatus(jniEnv,api,api->SessionGetOutputName((OrtSession*)sessionHandle, i, allocator, &outputName)); - jstring name = (*jniEnv)->NewStringUTF(jniEnv,outputName); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,outputName)); + // Get the number of outputs + size_t numOutputs = 0; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->SessionGetOutputCount(session, &numOutputs)); + if (code != ORT_OK) { + return NULL; + } - // Create a ValueInfo from the OrtTypeInfo - OrtTypeInfo* typeInfo; - checkOrtStatus(jniEnv,api,api->SessionGetOutputTypeInfo((OrtSession*)sessionHandle, i, &typeInfo)); - jobject valueInfoJava = convertToValueInfo(jniEnv,api,typeInfo); - api->ReleaseTypeInfo(typeInfo); - - // Create a NodeInfo and assign into the array - jobject nodeInfo = (*jniEnv)->NewObject(jniEnv, nodeInfoClazz, nodeInfoConstructor, name, valueInfoJava); - (*jniEnv)->SetObjectArrayElement(jniEnv, array, i, nodeInfo); + // Allocate the return array + jobjectArray array = (*jniEnv)->NewObjectArray(jniEnv, safecast_size_t_to_jsize(numOutputs), nodeInfoClazz, NULL); + for (uint32_t i = 0; i < numOutputs; i++) { + // Read out the output name and convert it to a java.lang.String + char* outputName = NULL; + code = checkOrtStatus(jniEnv, api, api->SessionGetOutputName(session, i, allocator, &outputName)); + if (code != ORT_OK) { + break; + } + jstring name = (*jniEnv)->NewStringUTF(jniEnv, outputName); + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, outputName)); + if (code != ORT_OK) { + break; } - return array; + // Create a ValueInfo from the OrtTypeInfo + OrtTypeInfo* typeInfo = NULL; + code = checkOrtStatus(jniEnv, api, api->SessionGetOutputTypeInfo(session, i, &typeInfo)); + if (code != ORT_OK) { + break; + } + jobject valueInfoJava = convertToValueInfo(jniEnv, api, typeInfo); + api->ReleaseTypeInfo(typeInfo); + if (valueInfoJava == NULL) { + break; + } + + // Create a NodeInfo and assign into the array + jobject nodeInfo = (*jniEnv)->NewObject(jniEnv, nodeInfoClazz, nodeInfoConstructor, name, valueInfoJava); + (*jniEnv)->SetObjectArrayElement(jniEnv, array, i, nodeInfo); + } + + return array; } /* @@ -242,99 +318,139 @@ JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_getOutputInfo * Signature: (JJJ[Ljava/lang/String;[JJ[Ljava/lang/String;JJ)[Lai/onnxruntime/OnnxValue; * private native OnnxValue[] run(long apiHandle, long nativeHandle, long allocatorHandle, String[] inputNamesArray, long[] inputs, long numInputs, String[] outputNamesArray, long numOutputs) */ -JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_run - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle, jlong allocatorHandle, jobjectArray inputNamesArr, jlongArray tensorArr, jlong numInputs, jobjectArray outputNamesArr, jlong numOutputs, jlong runOptionsHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; - OrtSession* session = (OrtSession*) sessionHandle; - OrtRunOptions* runOptions = (OrtRunOptions*) runOptionsHandle; +JNIEXPORT jobjectArray JNICALL Java_ai_onnxruntime_OrtSession_run(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, + jlong sessionHandle, jlong allocatorHandle, + jobjectArray inputNamesArr, jlongArray tensorArr, + jlong numInputs, jobjectArray outputNamesArr, + jlong numOutputs, jlong runOptionsHandle) { - // Create the buffers for the Java input and output strings - const char** inputNames; - checkOrtStatus(jniEnv, api, api->AllocatorAlloc(allocator,sizeof(char*)*numInputs,(void**)&inputNames)); - const char** outputNames; - checkOrtStatus(jniEnv, api, api->AllocatorAlloc(allocator,sizeof(char*)*numOutputs,(void**)&outputNames)); - jobject* javaInputStrings; - checkOrtStatus(jniEnv, api, api->AllocatorAlloc(allocator,sizeof(jobject)*numInputs,(void**)&javaInputStrings)); - jobject* javaOutputStrings; - checkOrtStatus(jniEnv, api, api->AllocatorAlloc(allocator,sizeof(jobject)*numOutputs,(void**)&javaOutputStrings)); + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; + OrtSession* session = (OrtSession*)sessionHandle; + OrtRunOptions* runOptions = (OrtRunOptions*)runOptionsHandle; - // Extract a C array of longs which are pointers to the input tensors. - // Need to convert longs to OrtValue* in case we run on non-64bit systems - jlong* inputTensors = (*jniEnv)->GetLongArrayElements(jniEnv,tensorArr,NULL); - const OrtValue** inputValues; - checkOrtStatus(jniEnv, api, api->AllocatorAlloc(allocator,sizeof(OrtValue*)*numInputs,(void**)&inputValues)); - - // Extract the names and native pointers of the input values. - for (int i = 0; i < numInputs; i++) { - javaInputStrings[i] = (*jniEnv)->GetObjectArrayElement(jniEnv,inputNamesArr,i); - inputNames[i] = (*jniEnv)->GetStringUTFChars(jniEnv,javaInputStrings[i],NULL); - inputValues[i] = (OrtValue*)inputTensors[i]; - } - - // Extract the names of the output values, and allocate their output array. - OrtValue** outputValues; - checkOrtStatus(jniEnv,api,api->AllocatorAlloc(allocator,sizeof(OrtValue*)*numOutputs,(void**)&outputValues)); - for (int i = 0; i < numOutputs; i++) { - javaOutputStrings[i] = (*jniEnv)->GetObjectArrayElement(jniEnv,outputNamesArr,i); - outputNames[i] = (*jniEnv)->GetStringUTFChars(jniEnv,javaOutputStrings[i],NULL); - outputValues[i] = NULL; - } - - // Actually score the inputs. - //printf("inputTensors = %p, first tensor = %p, numInputs = %ld, outputValues = %p, numOutputs = %ld\n",inputTensors,(OrtValue*)inputTensors[0],numInputs,outputValues,numOutputs); - //ORT_API_STATUS(OrtRun, _Inout_ OrtSession* sess, _In_ OrtRunOptions* run_options, _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, _In_ const char* const* output_names, size_t output_names_len, _Out_ OrtValue** output); - checkOrtStatus(jniEnv,api,api->Run(session, runOptions, (const char* const*) inputNames, (const OrtValue* const*) inputValues, numInputs, (const char* const*) outputNames, numOutputs, outputValues)); - // Release the C array of pointers to the tensors. - (*jniEnv)->ReleaseLongArrayElements(jniEnv,tensorArr,inputTensors,JNI_ABORT); - - // Construct the output array of ONNXValues - char *onnxValueClassName = "ai/onnxruntime/OnnxValue"; - jclass onnxValueClass = (*jniEnv)->FindClass(jniEnv, onnxValueClassName); - jobjectArray outputArray = (*jniEnv)->NewObjectArray(jniEnv,safecast_int64_to_jsize(numOutputs), onnxValueClass, NULL); - - // Convert the output tensors into ONNXValues and release the output strings. - for (int i = 0; i < numOutputs; i++) { - if (outputValues[i] != NULL) { - jobject onnxValue = convertOrtValueToONNXValue(jniEnv,api,allocator,outputValues[i]); - (*jniEnv)->SetObjectArrayElement(jniEnv,outputArray,i,onnxValue); - } - (*jniEnv)->ReleaseStringUTFChars(jniEnv,javaOutputStrings[i],outputNames[i]); - } - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,outputValues)); - - // Release the Java input strings - for (int i = 0; i < numInputs; i++) { - (*jniEnv)->ReleaseStringUTFChars(jniEnv,javaInputStrings[i],inputNames[i]); - } - - // Release the buffers - checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, (void*)inputNames)); - checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, (void*)inputValues)); - checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, (void*)outputNames)); - checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, javaInputStrings)); - checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, javaOutputStrings)); + jobjectArray outputArray = NULL; + // Create the buffers for the Java input & output strings, and the input pointers + const char** inputNames = malloc(sizeof(char*) * numInputs); + if (inputNames == NULL) { + // Nothing to cleanup, return and throw exception return outputArray; -} + } + const char** outputNames = malloc(sizeof(char*) * numOutputs); + if (outputNames == NULL) { + goto cleanup_input_names; + } + jobject* javaInputStrings = malloc(sizeof(jobject) * numInputs); + if (javaInputStrings == NULL) { + goto cleanup_output_names; + } + jobject* javaOutputStrings = malloc(sizeof(jobject) * numOutputs); + if (javaOutputStrings == NULL) { + goto cleanup_java_input_strings; + } + const OrtValue** inputValuePtrs = malloc(sizeof(OrtValue*) * numInputs); + if (inputValuePtrs == NULL) { + goto cleanup_java_output_strings; + } + OrtValue** outputValues = malloc(sizeof(OrtValue*) * numOutputs); + if (outputValues == NULL) { + goto cleanup_input_values; + } + // Extract a C array of longs which are pointers to the input tensors. + // The Java-side objects store native pointers as 64-bit longs, and on 32-bit systems + // we cannot cast the long array to a pointer array as they are different sizes, + // so we copy the longs applying the appropriate cast. + jlong* inputValueLongs = (*jniEnv)->GetLongArrayElements(jniEnv, tensorArr, NULL); + + // Extract the names and native pointers of the input values. + for (int i = 0; i < numInputs; i++) { + javaInputStrings[i] = (*jniEnv)->GetObjectArrayElement(jniEnv, inputNamesArr, i); + inputNames[i] = (*jniEnv)->GetStringUTFChars(jniEnv, javaInputStrings[i], NULL); + inputValuePtrs[i] = (OrtValue*)inputValueLongs[i]; + } + + // Release the java array copy of pointers to the tensors. + (*jniEnv)->ReleaseLongArrayElements(jniEnv, tensorArr, inputValueLongs, JNI_ABORT); + + // Extract the names of the output values. + for (int i = 0; i < numOutputs; i++) { + javaOutputStrings[i] = (*jniEnv)->GetObjectArrayElement(jniEnv, outputNamesArr, i); + outputNames[i] = (*jniEnv)->GetStringUTFChars(jniEnv, javaOutputStrings[i], NULL); + outputValues[i] = NULL; + } + + // Actually score the inputs. + // ORT_API_STATUS(OrtRun, _Inout_ OrtSession* sess, _In_ OrtRunOptions* run_options, + // _In_ const char* const* input_names, _In_ const OrtValue* const* input, size_t input_len, + // _In_ const char* const* output_names, size_t output_names_len, _Out_ OrtValue** output); + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->Run(session, runOptions, (const char* const*)inputNames, + (const OrtValue* const*)inputValuePtrs, numInputs, + (const char* const*)outputNames, numOutputs, outputValues)); + if (code != ORT_OK) { + goto cleanup_output_values; + } + + // Construct the output array of ONNXValues + jclass onnxValueClass = (*jniEnv)->FindClass(jniEnv, ORTJNI_OnnxValueClassName); + outputArray = (*jniEnv)->NewObjectArray(jniEnv, safecast_int64_to_jsize(numOutputs), onnxValueClass, NULL); + + // Convert the output tensors into ONNXValues + for (int i = 0; i < numOutputs; i++) { + if (outputValues[i] != NULL) { + jobject onnxValue = convertOrtValueToONNXValue(jniEnv, api, allocator, outputValues[i]); + if (onnxValue == NULL) { + break; // go to cleanup, exception thrown + } + (*jniEnv)->SetObjectArrayElement(jniEnv, outputArray, i, onnxValue); + } + } + + // Note these gotos are in a specific order so they mirror the allocation pattern above. + // They must be changed if the allocation code is rearranged. +cleanup_output_values: + free(outputValues); + + // Release the Java output strings + for (int i = 0; i < numOutputs; i++) { + (*jniEnv)->ReleaseStringUTFChars(jniEnv, javaOutputStrings[i], outputNames[i]); + } + + // Release the Java input strings + for (int i = 0; i < numInputs; i++) { + (*jniEnv)->ReleaseStringUTFChars(jniEnv, javaInputStrings[i], inputNames[i]); + } + + // Release the buffers +cleanup_input_values: + free((void*)inputValuePtrs); +cleanup_java_output_strings: + free(javaOutputStrings); +cleanup_java_input_strings: + free(javaInputStrings); +cleanup_output_names: + free((void*)outputNames); +cleanup_input_names: + free((void*)inputNames); + + return outputArray; +} /* * Class: ai_onnxruntime_OrtSession * Method: getProfilingStartTimeInNs * Signature: (JJ)J */ -JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getProfilingStartTimeInNs - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtSession* session = (OrtSession*) sessionHandle; +JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getProfilingStartTimeInNs(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong sessionHandle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtSession* session = (OrtSession*)sessionHandle; uint64_t timestamp = 0; - - checkOrtStatus(jniEnv,api,api->SessionGetProfilingStartTimeNs(session,×tamp)); - return (jlong) timestamp; + checkOrtStatus(jniEnv, api, api->SessionGetProfilingStartTimeNs(session, ×tamp)); + return (jlong)timestamp; } /* @@ -342,16 +458,19 @@ JNIEXPORT jlong JNICALL Java_ai_onnxruntime_OrtSession_getProfilingStartTimeInNs * Method: endProfiling * Signature: (JJJ)Ljava/lang/String; */ -JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtSession_endProfiling - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle, jlong allocatorHandle) { - (void) jobj; // Required JNI parameters not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; +JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtSession_endProfiling(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong handle, jlong allocatorHandle) { + (void)jobj; // Required JNI parameters not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; - char* profileStr; - checkOrtStatus(jniEnv,api,api->SessionEndProfiling((OrtSession*)handle,allocator,&profileStr)); - jstring profileOutput = (*jniEnv)->NewStringUTF(jniEnv,profileStr); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,profileStr)); + char* profileStr = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->SessionEndProfiling((OrtSession*)handle, allocator, + &profileStr)); + if (code != ORT_OK) { + return NULL; + } + jstring profileOutput = (*jniEnv)->NewStringUTF(jniEnv, profileStr); + checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, profileStr)); return profileOutput; } @@ -360,11 +479,10 @@ JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtSession_endProfiling * Method: closeSession * Signature: (J)V */ -JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_closeSession - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong handle) { - (void) jniEnv; (void) jobj; // Required JNI parameters not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - api->ReleaseSession((OrtSession*)handle); +JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_closeSession(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong handle) { + (void)jniEnv; (void)jobj; // Required JNI parameters not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + api->ReleaseSession((OrtSession*)handle); } /* @@ -372,91 +490,139 @@ JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_closeSession * Method: constructMetadata * Signature: (JJJ)Ljava/lang/String; */ -JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtSession_constructMetadata - (JNIEnv * jniEnv, jobject jobj, jlong apiHandle, jlong nativeHandle, jlong allocatorHandle) { - (void) jobj; // Required JNI parameter not needed by functions which don't need to access their host object. - const OrtApi* api = (const OrtApi*) apiHandle; - OrtAllocator* allocator = (OrtAllocator*) allocatorHandle; +JNIEXPORT jstring JNICALL Java_ai_onnxruntime_OrtSession_constructMetadata(JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong nativeHandle, jlong allocatorHandle) { + (void)jobj; // Required JNI parameter not needed by functions which don't need to access their host object. + const OrtApi* api = (const OrtApi*)apiHandle; + OrtAllocator* allocator = (OrtAllocator*)allocatorHandle; + jobject metadataJava = NULL; + jstring producerStr = NULL; + jstring graphStr = NULL; + jstring graphDescStr = NULL; + jstring domainStr = NULL; + jstring descriptionStr = NULL; + + // macro for processing char* into a Java UTF-8 string with error handling inside this function +#define STR_PROCESS(STR_NAME) \ + if (code == ORT_OK) { \ + STR_NAME = (*jniEnv)->NewStringUTF(jniEnv, charBuffer); \ + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, charBuffer)); \ + if (code != ORT_OK) { \ + goto release_metadata; \ + } \ + } else { \ + goto release_metadata; \ + } // Setup - char* stringClassName = "java/lang/String"; - jclass stringClazz = (*jniEnv)->FindClass(jniEnv, stringClassName); - char *metadataClassName = "ai/onnxruntime/OnnxModelMetadata"; - jclass metadataClazz = (*jniEnv)->FindClass(jniEnv, metadataClassName); - //OnnxModelMetadata(String producerName, String graphName, String domain, String description, long version, String[] customMetadataArray) - jmethodID metadataConstructor = (*jniEnv)->GetMethodID(jniEnv, metadataClazz, "", - "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;J[Ljava/lang/String;)V"); + jclass stringClazz = (*jniEnv)->FindClass(jniEnv, ORTJNI_StringClassName); + jclass metadataClazz = (*jniEnv)->FindClass(jniEnv, ORTJNI_MetadataClassName); + // OnnxModelMetadata(String producerName, String graphName, String domain, String description, + // long version, String[] customMetadataArray) + jmethodID metadataConstructor = (*jniEnv)->GetMethodID( + jniEnv, metadataClazz, "", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;J[Ljava/lang/String;)V"); // Get metadata - OrtModelMetadata* metadata; - checkOrtStatus(jniEnv,api,api->SessionGetModelMetadata((OrtSession*)nativeHandle,&metadata)); + OrtModelMetadata* metadata = NULL; + OrtErrorCode code = checkOrtStatus(jniEnv, api, api->SessionGetModelMetadata((OrtSession*)nativeHandle, &metadata)); + if (code != ORT_OK) { + // Nothing to cleanup, return null as an exception has been thrown + return NULL; + } // Read out the producer name and convert it to a java.lang.String - char* charBuffer; - checkOrtStatus(jniEnv,api,api->ModelMetadataGetProducerName(metadata, allocator, &charBuffer)); - jstring producerStr = (*jniEnv)->NewStringUTF(jniEnv,charBuffer); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,charBuffer)); + char* charBuffer = NULL; + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetProducerName(metadata, allocator, &charBuffer)); + STR_PROCESS(producerStr) // Read out the graph name and convert it to a java.lang.String - checkOrtStatus(jniEnv,api,api->ModelMetadataGetGraphName(metadata, allocator, &charBuffer)); - jstring graphStr = (*jniEnv)->NewStringUTF(jniEnv,charBuffer); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,charBuffer)); + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetGraphName(metadata, allocator, &charBuffer)); + STR_PROCESS(graphStr) // Read out the graph description and convert it to a java.lang.String - checkOrtStatus(jniEnv,api,api->ModelMetadataGetGraphDescription(metadata, allocator, &charBuffer)); - jstring graphDescStr = (*jniEnv)->NewStringUTF(jniEnv,charBuffer); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,charBuffer)); + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetGraphDescription(metadata, allocator, &charBuffer)); + STR_PROCESS(graphDescStr) // Read out the domain and convert it to a java.lang.String - checkOrtStatus(jniEnv,api,api->ModelMetadataGetDomain(metadata, allocator, &charBuffer)); - jstring domainStr = (*jniEnv)->NewStringUTF(jniEnv,charBuffer); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,charBuffer)); + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetDomain(metadata, allocator, &charBuffer)); + STR_PROCESS(domainStr) // Read out the description and convert it to a java.lang.String - checkOrtStatus(jniEnv,api,api->ModelMetadataGetDescription(metadata, allocator, &charBuffer)); - jstring descriptionStr = (*jniEnv)->NewStringUTF(jniEnv,charBuffer); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,charBuffer)); + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetDescription(metadata, allocator, &charBuffer)); + STR_PROCESS(descriptionStr) // Read out the version - int64_t version; - checkOrtStatus(jniEnv,api,api->ModelMetadataGetVersion(metadata, &version)); + int64_t version = 0; + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetVersion(metadata, &version)); + if (code != ORT_OK) { + goto release_metadata; + } // Read out the keys, look up the values. - int64_t numKeys; - char** keys; - checkOrtStatus(jniEnv,api,api->ModelMetadataGetCustomMetadataMapKeys(metadata, allocator, &keys, &numKeys)); + int64_t numKeys = 0; + char** keys = NULL; + code = checkOrtStatus(jniEnv, api, api->ModelMetadataGetCustomMetadataMapKeys(metadata, allocator, &keys, &numKeys)); + if (code != ORT_OK) { + goto release_metadata; + } jobjectArray customArray = NULL; if (numKeys > 0) { - customArray = (*jniEnv)->NewObjectArray(jniEnv,safecast_int64_to_jsize(numKeys * 2),stringClazz, NULL); + customArray = (*jniEnv)->NewObjectArray(jniEnv, safecast_int64_to_jsize(numKeys * 2), stringClazz, NULL); // Iterate key array to extract the values for (int64_t i = 0; i < numKeys; i++) { // Create a java.lang.String for the key - jstring keyJava = (*jniEnv)->NewStringUTF(jniEnv,keys[i]); + jstring keyJava = (*jniEnv)->NewStringUTF(jniEnv, keys[i]); // Extract the value and convert it to a java.lang.String - checkOrtStatus(jniEnv,api,api->ModelMetadataLookupCustomMetadataMap(metadata,allocator,keys[i],&charBuffer)); - jstring valueJava = (*jniEnv)->NewStringUTF(jniEnv,charBuffer); - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,charBuffer)); + code = checkOrtStatus(jniEnv, api, api->ModelMetadataLookupCustomMetadataMap(metadata, allocator, keys[i], &charBuffer)); + jstring valueJava = NULL; + if (code == ORT_OK) { + valueJava = (*jniEnv)->NewStringUTF(jniEnv, charBuffer); + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, charBuffer)); + if (code != ORT_OK) { + // Signal that custom metadata extraction failed and break out + customArray = NULL; + break; + } + } else { + // Signal that custom metadata extraction failed and break out + customArray = NULL; + break; + } // Write the key and value into the array - (*jniEnv)->SetObjectArrayElement(jniEnv,customArray,safecast_int64_to_jsize(i*2),keyJava); - (*jniEnv)->SetObjectArrayElement(jniEnv,customArray,safecast_int64_to_jsize((i * 2) + 1),valueJava); + (*jniEnv)->SetObjectArrayElement(jniEnv, customArray, safecast_int64_to_jsize(i * 2), keyJava); + (*jniEnv)->SetObjectArrayElement(jniEnv, customArray, safecast_int64_to_jsize((i * 2) + 1), valueJava); } // Release key array for (int64_t i = 0; i < numKeys; i++) { - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,keys[i])); + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, keys[i])); + if (code != ORT_OK) { + customArray = NULL; + break; + } + } + code = checkOrtStatus(jniEnv, api, api->AllocatorFree(allocator, keys)); + if (code != ORT_OK) { + customArray = NULL; } - checkOrtStatus(jniEnv,api,api->AllocatorFree(allocator,keys)); } else { - customArray = (*jniEnv)->NewObjectArray(jniEnv,0,stringClazz,NULL); + customArray = (*jniEnv)->NewObjectArray(jniEnv, 0, stringClazz, NULL); } - // Invoke the metadata constructor - //OnnxModelMetadata(String producerName, String graphName, String graphDescription, String domain, String description, long version, String[] customMetadataArray) - jobject metadataJava = (*jniEnv)->NewObject(jniEnv, metadataClazz, metadataConstructor, producerStr, graphStr, graphDescStr, domainStr, descriptionStr, (jlong) version, customArray); + if (customArray != NULL) { + // If the array is non-null then the custom metadata extraction completed successfully so + // we invoke the metadata constructor + // OnnxModelMetadata(String producerName, String graphName, String graphDescription, String domain, + // String description, long version, String[] customMetadataArray) + metadataJava = (*jniEnv)->NewObject(jniEnv, metadataClazz, metadataConstructor, + producerStr, graphStr, graphDescStr, domainStr, descriptionStr, (jlong)version, + customArray); + } +release_metadata: // Release the metadata api->ReleaseModelMetadata(metadata); diff --git a/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c b/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c index b880d1c4cb..a4b336d745 100644 --- a/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c +++ b/java/src/main/native/ai_onnxruntime_OrtSession_SessionOptions.c @@ -4,6 +4,7 @@ */ #include #include +#include #include "onnxruntime/core/session/onnxruntime_c_api.h" #include "OrtJniUtil.h" #include "ai_onnxruntime_OrtSession_SessionOptions.h" @@ -615,3 +616,43 @@ JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addROC throwOrtException(jniEnv,convertErrorCode(ORT_INVALID_ARGUMENT),"This binary was not compiled with ROCM support."); #endif } + +/* + * Class:: ai_onnxruntime_OrtSession_SessionOptions + * Method: addExecutionProvider + * Signature: (JILjava/lang/String)V + */ +JNIEXPORT void JNICALL Java_ai_onnxruntime_OrtSession_00024SessionOptions_addExecutionProvider( + JNIEnv* jniEnv, jobject jobj, jlong apiHandle, jlong optionsHandle, + jstring jepName, jobjectArray configKeyArr, jobjectArray configValueArr) { + (void)jobj; + + const char* epName = (*jniEnv)->GetStringUTFChars(jniEnv, jepName, NULL); + const OrtApi* api = (const OrtApi*)apiHandle; + OrtSessionOptions* options = (OrtSessionOptions*)optionsHandle; + int keyCount = (*jniEnv)->GetArrayLength(jniEnv, configKeyArr); + + const char** keyArray = (const char**)malloc(keyCount * sizeof(const char*)); + const char** valueArray = (const char**)malloc(keyCount * sizeof(const char*)); + jstring* jkeyArray = (jstring*)malloc(keyCount * sizeof(jstring)); + jstring* jvalueArray = (jstring*)malloc(keyCount * sizeof(jstring)); + + for (int i = 0; i < keyCount; i++) { + jkeyArray[i] = (jstring)((*jniEnv)->GetObjectArrayElement(jniEnv, configKeyArr, i)); + jvalueArray[i] = (jstring)((*jniEnv)->GetObjectArrayElement(jniEnv, configValueArr, i)); + keyArray[i] = (*jniEnv)->GetStringUTFChars(jniEnv, jkeyArray[i], NULL); + valueArray[i] = (*jniEnv)->GetStringUTFChars(jniEnv, jvalueArray[i], NULL); + } + + checkOrtStatus(jniEnv, api, api->SessionOptionsAppendExecutionProvider(options, epName, keyArray, valueArray, keyCount)); + + for (int i = 0; i < keyCount; i++) { + (*jniEnv)->ReleaseStringUTFChars(jniEnv, jkeyArray[i], keyArray[i]); + (*jniEnv)->ReleaseStringUTFChars(jniEnv, jvalueArray[i], valueArray[i]); + } + (*jniEnv)->ReleaseStringUTFChars(jniEnv, jepName, epName); + free((void*)keyArray); + free((void*)valueArray); + free((void*)jkeyArray); + free((void*)jvalueArray); +} diff --git a/java/src/test/android/README.md b/java/src/test/android/README.md index f1bacfeb29..c5658be386 100644 --- a/java/src/test/android/README.md +++ b/java/src/test/android/README.md @@ -1,4 +1,4 @@ -# Android Test Application for ORT-Mobile +# Android Test Application for ORT-Mobile This directory contains a simple android application for testing [ONNX Runtime AAR package](https://www.onnxruntime.ai/docs/how-to/build.html#build-android-archive-aar). @@ -11,7 +11,7 @@ For general usage and build purpose of ORT-Mobile Android, please see the [docum This android application is mainly aimed for testing: - Model used: A simple [sigmoid ONNX model](https://github.com/onnx/onnx/blob/f9b0cc99344869c246b8f4011b8586a39841284c/onnx/backend/test/data/node/test_sigmoid/model.onnx) (converted to ORT format under `app\src\androidTest\assets` folder). - - Here's a [documentation](https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_for_Mobile_Platforms.md#1-create-ort-format-model-and-configuration-file-with-required-operators) about how you can convert an ONNX model into ORT format. + - Here's a [documentation](https://github.com/microsoft/onnxruntime/blob/main/docs/ONNX_Runtime_for_Mobile_Platforms.md#1-create-ort-format-model-and-configuration-file-with-required-operators) about how you can convert an ONNX model into ORT format. - Main test file: An android instrumentation test under `app\src\androidtest\java\ai.onnxruntime.example.javavalidator\SimpleTest.kt` - The main dependency of this application is `onnxruntime` aar package under `app\libs`. - The MainActivity of this application is set to be empty. @@ -31,7 +31,7 @@ Please note that you may need to set the `--android_abi=x86_64` (the default opt The build will generate two apks which is required to run the test application in `$YOUR_BUILD_DIR/java/androidtest/android/app/build/outputs/apk`: -* `androidtest/debug/app-debug-androidtest.apk` +* `androidtest/debug/app-debug-androidtest.apk` * `debug/app-debug.apk` After running the build script, the two apks will be installed on `ort_android` emulator and it will automatically run the test application in an adb shell. diff --git a/java/src/test/java/ai/onnxruntime/InferenceTest.java b/java/src/test/java/ai/onnxruntime/InferenceTest.java index 9c22f77f72..7b11b71c64 100644 --- a/java/src/test/java/ai/onnxruntime/InferenceTest.java +++ b/java/src/test/java/ai/onnxruntime/InferenceTest.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -617,6 +618,12 @@ public class InferenceTest { runProvider(OrtProvider.DNNL); } + @Test + @EnabledIfSystemProperty(named = "USE_XNNPACK", matches = "1") + public void testXNNPACK() throws OrtException { + runProvider(OrtProvider.XNNPACK); + } + private void runProvider(OrtProvider provider) throws OrtException { EnumSet providers = OrtEnvironment.getAvailableProviders(); assertTrue(providers.size() > 1); @@ -1515,6 +1522,9 @@ public class InferenceTest { case CORE_ML: options.addCoreML(); break; + case XNNPACK: + options.addXnnpack(Collections.emptyMap()); + break; case NUPHAR: options.addNuphar(true, ""); break; diff --git a/java/src/test/java/ai/onnxruntime/OnnxMl.java b/java/src/test/java/ai/onnxruntime/OnnxMl.java index eb2f0121a8..d96c6ccba6 100644 --- a/java/src/test/java/ai/onnxruntime/OnnxMl.java +++ b/java/src/test/java/ai/onnxruntime/OnnxMl.java @@ -24951,7 +24951,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -24965,7 +24965,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25177,7 +25177,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25201,7 +25201,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25742,7 +25742,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25766,7 +25766,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25790,7 +25790,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25812,7 +25812,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -25831,7 +25831,7 @@ public final class OnnxMl { * Standard denotation can optionally be used to denote tensor * dimensions with standard semantic descriptions to ensure * that operations are applied to the correct axis of a tensor. - * Refer to https://github.com/onnx/onnx/blob/master/docs/DimensionDenotation.md#denotation-definition + * Refer to https://github.com/onnx/onnx/blob/main/docs/DimensionDenotation.md#denotation-definition * for pre-defined dimension denotations. * * @@ -26657,7 +26657,7 @@ public final class OnnxMl { * 

            * An optional denotation can be used to denote the whole
      * type with a standard semantic description as to what is
-     * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+     * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
      * for pre-defined type denotations.
      *
      * @@ -26670,7 +26670,7 @@ public final class OnnxMl { * 
            * An optional denotation can be used to denote the whole
      * type with a standard semantic description as to what is
-     * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+     * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
      * for pre-defined type denotations.
      *
      * @@ -30755,7 +30755,7 @@ public final class OnnxMl { * 
            * An optional denotation can be used to denote the whole
      * type with a standard semantic description as to what is
-     * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+     * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
      * for pre-defined type denotations.
      *
      * @@ -30778,7 +30778,7 @@ public final class OnnxMl { * 
            * An optional denotation can be used to denote the whole
      * type with a standard semantic description as to what is
-     * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+     * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
      * for pre-defined type denotations.
      *
      * @@ -32122,7 +32122,7 @@ public final class OnnxMl { * 
              * An optional denotation can be used to denote the whole
        * type with a standard semantic description as to what is
-       * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+       * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
        * for pre-defined type denotations.
        *
      * @@ -32145,7 +32145,7 @@ public final class OnnxMl { * 
              * An optional denotation can be used to denote the whole
        * type with a standard semantic description as to what is
-       * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+       * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
        * for pre-defined type denotations.
        *
      * @@ -32168,7 +32168,7 @@ public final class OnnxMl { * 
              * An optional denotation can be used to denote the whole
        * type with a standard semantic description as to what is
-       * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+       * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
        * for pre-defined type denotations.
        *
      * @@ -32189,7 +32189,7 @@ public final class OnnxMl { * 
              * An optional denotation can be used to denote the whole
        * type with a standard semantic description as to what is
-       * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+       * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
        * for pre-defined type denotations.
        *
      * @@ -32207,7 +32207,7 @@ public final class OnnxMl { * 
              * An optional denotation can be used to denote the whole
        * type with a standard semantic description as to what is
-       * stored inside. Refer to https://github.com/onnx/onnx/blob/master/docs/TypeDenotation.md#type-denotation-definition
+       * stored inside. Refer to https://github.com/onnx/onnx/blob/main/docs/TypeDenotation.md#type-denotation-definition
        * for pre-defined type denotations.
        *
      * diff --git a/js/common/README.md b/js/common/README.md index 2f79561c32..b3768f3e38 100644 --- a/js/common/README.md +++ b/js/common/README.md @@ -10,4 +10,4 @@ This package (onnxruntime-common) is not designed for using directly. Please con ## License -License information can be found [here](https://github.com/microsoft/onnxruntime/blob/master/README.md#license). +License information can be found [here](https://github.com/microsoft/onnxruntime/blob/main/README.md#license). diff --git a/js/common/lib/inference-session.ts b/js/common/lib/inference-session.ts index 6e0faafbe8..06864f6a2a 100644 --- a/js/common/lib/inference-session.ts +++ b/js/common/lib/inference-session.ts @@ -117,7 +117,7 @@ export declare namespace InferenceSession { /** * Log severity level. See - * https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/common/logging/severity.h + * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/common/logging/severity.h * * This setting is available only in ONNXRuntime (Node.js binding and react-native) or WebAssembly backend */ @@ -132,7 +132,7 @@ export declare namespace InferenceSession { /** * Store configurations for a session. See - * https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/ + * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/ * onnxruntime_session_options_config_keys.h * * This setting is available only in WebAssembly backend. Will support Node.js binding and react-native later @@ -201,7 +201,7 @@ export declare namespace InferenceSession { export interface RunOptions { /** * Log severity level. See - * https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/common/logging/severity.h + * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/common/logging/severity.h * * This setting is available only in ONNXRuntime (Node.js binding and react-native) or WebAssembly backend */ @@ -230,7 +230,7 @@ export declare namespace InferenceSession { /** * Set a single run configuration entry. See - * https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/ + * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/ * onnxruntime_run_options_config_keys.h * * This setting is available only in WebAssembly backend. Will support Node.js binding and react-native later diff --git a/js/node/README.md b/js/node/README.md index d2f616696c..24995e0012 100644 --- a/js/node/README.md +++ b/js/node/README.md @@ -26,4 +26,4 @@ To use on platforms without pre-built binaries, you can build Node.js binding fr ## License -License information can be found [here](https://github.com/microsoft/onnxruntime/blob/master/README.md#license). +License information can be found [here](https://github.com/microsoft/onnxruntime/blob/main/README.md#license). diff --git a/js/node/package-lock.json b/js/node/package-lock.json index f7d4a68a85..2ce0603cd9 100644 --- a/js/node/package-lock.json +++ b/js/node/package-lock.json @@ -18,13 +18,13 @@ }, "devDependencies": { "@types/fs-extra": "^9.0.6", - "@types/minimist": "1.2.1", + "@types/minimist": "^1.2.2", "@types/mocha": "^8.2.2", "@types/node": "^14.14.37", "cmake-js": "^6.2.1", "fs-extra": "^9.1.0", "jsonc": "^2.0.0", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mocha": "^9.2.1", "node-addon-api": "^3.1.0", "onnx-proto": "^4.0.4", @@ -1855,9 +1855,9 @@ "dev": true }, "node_modules/@types/minimist": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.1.tgz", - "integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", "dev": true }, "node_modules/@types/mocha": { @@ -3880,9 +3880,9 @@ "dev": true }, "@types/minimist": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.1.tgz", - "integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", "dev": true }, "@types/mocha": { diff --git a/js/node/package.json b/js/node/package.json index f44742ac2f..02fe5e1164 100644 --- a/js/node/package.json +++ b/js/node/package.json @@ -35,13 +35,13 @@ ], "devDependencies": { "@types/fs-extra": "^9.0.6", - "@types/minimist": "1.2.1", + "@types/minimist": "^1.2.2", "@types/mocha": "^8.2.2", "@types/node": "^14.14.37", "cmake-js": "^6.2.1", "fs-extra": "^9.1.0", "jsonc": "^2.0.0", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mocha": "^9.2.1", "node-addon-api": "^3.1.0", "onnx-proto": "^4.0.4", diff --git a/js/react_native/README.md b/js/react_native/README.md index 9fdc4a5fe9..d64d08484a 100644 --- a/js/react_native/README.md +++ b/js/react_native/README.md @@ -38,4 +38,4 @@ ONNX Runtime React Native currently supports most operators used by popular mode ### License -License information can be found [here](https://github.com/microsoft/onnxruntime/blob/master/README.md#license). +License information can be found [here](https://github.com/microsoft/onnxruntime/blob/main/README.md#license). diff --git a/js/web/README.md b/js/web/README.md index e81cbc234c..9ab9a7dac7 100644 --- a/js/web/README.md +++ b/js/web/README.md @@ -46,12 +46,12 @@ Refer to the following links for development information: #### WebAssembly backend -ONNX Runtime Web currently support all operators in [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md) and [ai.onnx.ml](https://github.com/onnx/onnx/blob/master/docs/Operators-ml.md). +ONNX Runtime Web currently support all operators in [ai.onnx](https://github.com/onnx/onnx/blob/main/docs/Operators.md) and [ai.onnx.ml](https://github.com/onnx/onnx/blob/main/docs/Operators-ml.md). #### WebGL backend -ONNX Runtime Web currently supports a subset of operators in [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md) operator set. See [operators.md](./docs/operators.md) for a complete, detailed list of which ONNX operators are supported by WebGL backend. +ONNX Runtime Web currently supports a subset of operators in [ai.onnx](https://github.com/onnx/onnx/blob/main/docs/Operators.md) operator set. See [operators.md](./docs/operators.md) for a complete, detailed list of which ONNX operators are supported by WebGL backend. ## License -License information can be found [here](https://github.com/microsoft/onnxruntime/blob/master/README.md#license). +License information can be found [here](https://github.com/microsoft/onnxruntime/blob/main/README.md#license). diff --git a/js/web/docs/operators.md b/js/web/docs/operators.md index ab425c3df8..96798f0453 100644 --- a/js/web/docs/operators.md +++ b/js/web/docs/operators.md @@ -1,6 +1,6 @@ ## Operators Support Table -The following table shows [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md) operators from which onnx opset version are currently supported by ONNX Runtime Web. For example, `4-6, 8+` means ONNX Runtime Web currently support opset version 4 to 6, 8 and above. +The following table shows [ai.onnx](https://github.com/onnx/onnx/blob/main/docs/Operators.md) operators from which onnx opset version are currently supported by ONNX Runtime Web. For example, `4-6, 8+` means ONNX Runtime Web currently support opset version 4 to 6, 8 and above. See [Compatibility](../README.md#Compatibility) for a list of the supported platforms. @@ -8,181 +8,181 @@ See [Compatibility](../README.md#Compatibility) for a list of the supported plat | Operator | WebGl Backend | |:--------:|:-------------:| -| [Abs](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Abs) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Abs-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Abs-13) | -| [Acos](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Acos) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Acos-7) | -| [Acosh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Acosh) | | -| [Add](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Add) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Add-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Add-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Add-14) | -| [And](https://github.com/onnx/onnx/blob/master/docs/Operators.md#And) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#And-7) | -| [ArgMax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ArgMax) | | -| [ArgMin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ArgMin) | | -| [Asin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Asin) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Asin-7) | -| [Asinh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Asinh) | | -| [Atan](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Atan) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Atan-7) | -| [Atanh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Atanh) | | -| [AveragePool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#AveragePool) | [7-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#AveragePool-7), [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#AveragePool-10), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#AveragePool-11) | -| [BatchNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#BatchNormalization) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-7), [9-13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-9), [14](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-14), [15+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#BatchNormalization-15) | -| [Bernoulli](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Bernoulli) | | -| [BitShift](https://github.com/onnx/onnx/blob/master/docs/Operators.md#BitShift) | | -| [BlackmanWindow](https://github.com/onnx/onnx/blob/master/docs/Operators.md#BlackmanWindow) | | -| [Cast](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Cast) | [6-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Cast-6), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Cast-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Cast-13) | -| [CastLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#CastLike) | | -| [Ceil](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Ceil) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Ceil-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Ceil-13) | -| [Celu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Celu) | | -| [Clip](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Clip) | [6-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Clip-6), [11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Clip-11), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Clip-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Clip-13) | -| [Compress](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Compress) | | -| [Concat](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Concat) | [4-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-4), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-13) | -| [ConcatFromSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConcatFromSequence) | | -| [Constant](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Constant) | | -| [ConstantOfShape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConstantOfShape) | | -| [Conv](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Conv) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Conv-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Conv-11) | -| [ConvInteger](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConvInteger) | | -| [ConvTranspose](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ConvTranspose) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ConvTranspose-1), [11+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ConvTranspose-11) | -| [Cos](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Cos) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Cos-7) | -| [Cosh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Cosh) | | -| [CumSum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#CumSum) | | -| [DFT](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DFT) | | -| [DepthToSpace](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DepthToSpace) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#DepthToSpace-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#DepthToSpace-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#DepthToSpace-13) | -| [DequantizeLinear](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DequantizeLinear) | | -| [Det](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Det) | | -| [Div](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Div) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Div-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Div-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Div-14) | -| [Dropout](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Dropout) | [7-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-7), [10-11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-10), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Dropout-13) | -| [DynamicQuantizeLinear](https://github.com/onnx/onnx/blob/master/docs/Operators.md#DynamicQuantizeLinear) | | -| [Einsum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Einsum) | | -| [Elu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Elu) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Elu-6) | -| [Equal](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Equal) | [7-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Equal-7), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Equal-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Equal-13) | -| [Erf](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Erf) | | -| [Exp](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Exp) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Exp-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Exp-13) | -| [Expand](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Expand) | | -| [EyeLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#EyeLike) | | -| [Flatten](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Flatten) | [1-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-1), [9-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-9), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Flatten-13) | -| [Floor](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Floor) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Floor-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Floor-13) | -| [GRU](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GRU) | | -| [Gather](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gather) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gather-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gather-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gather-13) | -| [GatherElements](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GatherElements) | | -| [GatherND](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GatherND) | | -| [Gemm](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gemm) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-7), [9-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-9), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Gemm-13) | -| [GlobalAveragePool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GlobalAveragePool) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#GlobalAveragePool-1) | -| [GlobalLpPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GlobalLpPool) | | -| [GlobalMaxPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GlobalMaxPool) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#GlobalMaxPool-1) | -| [Greater](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Greater) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Greater-7), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Greater-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Greater-13) | -| [GreaterOrEqual](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GreaterOrEqual) | | -| [GridSample](https://github.com/onnx/onnx/blob/master/docs/Operators.md#GridSample) | | -| [HammingWindow](https://github.com/onnx/onnx/blob/master/docs/Operators.md#HammingWindow) | | -| [HannWindow](https://github.com/onnx/onnx/blob/master/docs/Operators.md#HannWindow) | | -| [HardSigmoid](https://github.com/onnx/onnx/blob/master/docs/Operators.md#HardSigmoid) | | -| [HardSwish](https://github.com/onnx/onnx/blob/master/docs/Operators.md#HardSwish) | | -| [Hardmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Hardmax) | | -| [Identity](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Identity) | [1-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-1), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-13), [14-15](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-14), [16+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Identity-16) | -| [If](https://github.com/onnx/onnx/blob/master/docs/Operators.md#If) | | -| [InstanceNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#InstanceNormalization) | [6+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#InstanceNormalization-6) | -| [IsInf](https://github.com/onnx/onnx/blob/master/docs/Operators.md#IsInf) | | -| [IsNaN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#IsNaN) | | -| [LRN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LRN) | | -| [LSTM](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM) | | -| [LayerNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LayerNormalization) | | -| [LeakyRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LeakyRelu) | [6-15](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LeakyRelu-6), [16+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#LeakyRelu-16) | -| [Less](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Less) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-7), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Less-13) | -| [LessOrEqual](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LessOrEqual) | | -| [Log](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Log) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Log-13) | -| [LogSoftmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LogSoftmax) | | -| [Loop](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Loop) | | -| [LpNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LpNormalization) | | -| [LpPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#LpPool) | | -| [MatMul](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MatMul) | [1-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MatMul-1), [9-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MatMul-9), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MatMul-13) | -| [MatMulInteger](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MatMulInteger) | | -| [Max](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Max) | | -| [MaxPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MaxPool) | [1-7](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-1), [8-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-8), [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-10), [11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-11), [12+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#MaxPool-12) | -| [MaxRoiPool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MaxRoiPool) | | -| [MaxUnpool](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MaxUnpool) | | -| [Mean](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Mean) | | -| [MeanVarianceNormalization](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MeanVarianceNormalization) | | -| [MelWeightMatrix](https://github.com/onnx/onnx/blob/master/docs/Operators.md#MelWeightMatrix) | | -| [Min](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Min) | | -| [Mod](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Mod) | | -| [Mul](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Mul) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Mul-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Mul-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Mul-14) | -| [Multinomial](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Multinomial) | | -| [Neg](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Neg) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Neg-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Neg-13) | -| [NegativeLogLikelihoodLoss](https://github.com/onnx/onnx/blob/master/docs/Operators.md#NegativeLogLikelihoodLoss) | | -| [NonMaxSuppression](https://github.com/onnx/onnx/blob/master/docs/Operators.md#NonMaxSuppression) | | -| [NonZero](https://github.com/onnx/onnx/blob/master/docs/Operators.md#NonZero) | | -| [Not](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Not) | [1+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Not-1) | -| [OneHot](https://github.com/onnx/onnx/blob/master/docs/Operators.md#OneHot) | | -| [Optional](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Optional) | | -| [OptionalGetElement](https://github.com/onnx/onnx/blob/master/docs/Operators.md#OptionalGetElement) | | -| [OptionalHasElement](https://github.com/onnx/onnx/blob/master/docs/Operators.md#OptionalHasElement) | | -| [Or](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Or) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Or-7) | -| [PRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#PRelu) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#PRelu-7), [9-15](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#PRelu-9), [16+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#PRelu-16) | -| [Pad](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Pad) | [2-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pad-2), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pad-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pad-13) | -| [Pow](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Pow) | [7-11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-7), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-12), [13-14](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-13), [15+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Pow-15) | -| [QLinearConv](https://github.com/onnx/onnx/blob/master/docs/Operators.md#QLinearConv) | | -| [QLinearMatMul](https://github.com/onnx/onnx/blob/master/docs/Operators.md#QLinearMatMul) | | -| [QuantizeLinear](https://github.com/onnx/onnx/blob/master/docs/Operators.md#QuantizeLinear) | | -| [RNN](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RNN) | | -| [RandomNormal](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomNormal) | | -| [RandomNormalLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomNormalLike) | | -| [RandomUniform](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomUniform) | | -| [RandomUniformLike](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RandomUniformLike) | | -| [Range](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Range) | | -| [Reciprocal](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Reciprocal) | | -| [ReduceL1](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceL1) | | -| [ReduceL2](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceL2) | | -| [ReduceLogSum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceLogSum) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceLogSum-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceLogSum-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceLogSum-13) | -| [ReduceLogSumExp](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceLogSumExp) | | -| [ReduceMax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceMax) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-1), [11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-11), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMax-13) | -| [ReduceMean](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceMean) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMean-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMean-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMean-13) | -| [ReduceMin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceMin) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-1), [11](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-11), [12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-12), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceMin-13) | -| [ReduceProd](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceProd) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceProd-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceProd-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceProd-13) | -| [ReduceSum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceSum) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSum-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSum-11) | -| [ReduceSumSquare](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReduceSumSquare) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#ReduceSumSquare-13) | -| [Relu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Relu) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-6), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Relu-14) | -| [Reshape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Reshape) | [5-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-5), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Reshape-14) | -| [Resize](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Resize) | [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-10), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Resize-13) | -| [ReverseSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ReverseSequence) | | -| [RoiAlign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#RoiAlign) | | -| [Round](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Round) | | -| [STFT](https://github.com/onnx/onnx/blob/master/docs/Operators.md#STFT) | | -| [Scan](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Scan) | | -| [Scatter](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Scatter) | | -| [ScatterElements](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ScatterElements) | | -| [ScatterND](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ScatterND) | | -| [Selu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Selu) | | -| [SequenceAt](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceAt) | | -| [SequenceConstruct](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceConstruct) | | -| [SequenceEmpty](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceEmpty) | | -| [SequenceErase](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceErase) | | -| [SequenceInsert](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceInsert) | | -| [SequenceLength](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceLength) | | -| [SequenceMap](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SequenceMap) | | -| [Shape](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Shape) | [1-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Shape-1), [13-14](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Shape-13), [15+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Shape-15) | -| [Shrink](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Shrink) | | -| [Sigmoid](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sigmoid) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sigmoid-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sigmoid-13) | -| [Sign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sign) | | -| [Sin](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sin) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sin-7) | -| [Sinh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sinh) | | -| [Size](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Size) | | -| [Slice](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Slice) | [1-9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-1), [10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-10), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Slice-13) | -| [Softmax](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-13) | -| [SoftmaxCrossEntropyLoss](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SoftmaxCrossEntropyLoss) | | -| [Softplus](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softplus) | | -| [Softsign](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softsign) | | -| [SpaceToDepth](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SpaceToDepth) | | -| [Split](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Split) | [2-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Split-2), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Split-11) | -| [SplitToSequence](https://github.com/onnx/onnx/blob/master/docs/Operators.md#SplitToSequence) | | -| [Sqrt](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sqrt) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sqrt-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sqrt-13) | -| [Squeeze](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Squeeze) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Squeeze-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Squeeze-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Squeeze-13) | -| [StringNormalizer](https://github.com/onnx/onnx/blob/master/docs/Operators.md#StringNormalizer) | | -| [Sub](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sub) | [7-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sub-7), [13](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sub-13), [14+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sub-14) | -| [Sum](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Sum) | [6-7](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sum-6), [8-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sum-8), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Sum-13) | -| [Tan](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Tan) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tan-7) | -| [Tanh](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Tanh) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tanh-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tanh-13) | -| [TfIdfVectorizer](https://github.com/onnx/onnx/blob/master/docs/Operators.md#TfIdfVectorizer) | | -| [ThresholdedRelu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#ThresholdedRelu) | | -| [Tile](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Tile) | [6-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tile-6), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Tile-13) | -| [TopK](https://github.com/onnx/onnx/blob/master/docs/Operators.md#TopK) | | -| [Transpose](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Transpose) | [1-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Transpose-1), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Transpose-13) | -| [Trilu](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Trilu) | | -| [Unique](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Unique) | | -| [Unsqueeze](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Unsqueeze) | [1-10](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-1), [11-12](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-11), [13+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Unsqueeze-13) | -| [Upsample](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Upsample) | [7-8](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-7), [9](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Upsample-9) | -| [Where](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Where) | | -| [Xor](https://github.com/onnx/onnx/blob/master/docs/Operators.md#Xor) | [7+](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Xor-7) | +| [Abs](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Abs) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Abs-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Abs-13) | +| [Acos](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Acos) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Acos-7) | +| [Acosh](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Acosh) | | +| [Add](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Add) | [7-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Add-7), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Add-13), [14+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Add-14) | +| [And](https://github.com/onnx/onnx/blob/main/docs/Operators.md#And) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#And-7) | +| [ArgMax](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ArgMax) | | +| [ArgMin](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ArgMin) | | +| [Asin](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Asin) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Asin-7) | +| [Asinh](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Asinh) | | +| [Atan](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Atan) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Atan-7) | +| [Atanh](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Atanh) | | +| [AveragePool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#AveragePool) | [7-9](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#AveragePool-7), [10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#AveragePool-10), [11+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#AveragePool-11) | +| [BatchNormalization](https://github.com/onnx/onnx/blob/main/docs/Operators.md#BatchNormalization) | [7-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#BatchNormalization-7), [9-13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#BatchNormalization-9), [14](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#BatchNormalization-14), [15+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#BatchNormalization-15) | +| [Bernoulli](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Bernoulli) | | +| [BitShift](https://github.com/onnx/onnx/blob/main/docs/Operators.md#BitShift) | | +| [BlackmanWindow](https://github.com/onnx/onnx/blob/main/docs/Operators.md#BlackmanWindow) | | +| [Cast](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Cast) | [6-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Cast-6), [9-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Cast-9), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Cast-13) | +| [CastLike](https://github.com/onnx/onnx/blob/main/docs/Operators.md#CastLike) | | +| [Ceil](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Ceil) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Ceil-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Ceil-13) | +| [Celu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Celu) | | +| [Clip](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Clip) | [6-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Clip-6), [11](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Clip-11), [12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Clip-12), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Clip-13) | +| [Compress](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Compress) | | +| [Concat](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Concat) | [4-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Concat-4), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Concat-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Concat-13) | +| [ConcatFromSequence](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ConcatFromSequence) | | +| [Constant](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Constant) | | +| [ConstantOfShape](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ConstantOfShape) | | +| [Conv](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Conv) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Conv-1), [11+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Conv-11) | +| [ConvInteger](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ConvInteger) | | +| [ConvTranspose](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ConvTranspose) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ConvTranspose-1), [11+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ConvTranspose-11) | +| [Cos](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Cos) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Cos-7) | +| [Cosh](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Cosh) | | +| [CumSum](https://github.com/onnx/onnx/blob/main/docs/Operators.md#CumSum) | | +| [DFT](https://github.com/onnx/onnx/blob/main/docs/Operators.md#DFT) | | +| [DepthToSpace](https://github.com/onnx/onnx/blob/main/docs/Operators.md#DepthToSpace) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#DepthToSpace-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#DepthToSpace-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#DepthToSpace-13) | +| [DequantizeLinear](https://github.com/onnx/onnx/blob/main/docs/Operators.md#DequantizeLinear) | | +| [Det](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Det) | | +| [Div](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Div) | [7-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Div-7), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Div-13), [14+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Div-14) | +| [Dropout](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Dropout) | [7-9](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Dropout-7), [10-11](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Dropout-10), [12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Dropout-12), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Dropout-13) | +| [DynamicQuantizeLinear](https://github.com/onnx/onnx/blob/main/docs/Operators.md#DynamicQuantizeLinear) | | +| [Einsum](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Einsum) | | +| [Elu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Elu) | [6+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Elu-6) | +| [Equal](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Equal) | [7-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-7), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Equal-13) | +| [Erf](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Erf) | | +| [Exp](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Exp) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Exp-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Exp-13) | +| [Expand](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Expand) | | +| [EyeLike](https://github.com/onnx/onnx/blob/main/docs/Operators.md#EyeLike) | | +| [Flatten](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Flatten) | [1-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Flatten-1), [9-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Flatten-9), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Flatten-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Flatten-13) | +| [Floor](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Floor) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Floor-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Floor-13) | +| [GRU](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GRU) | | +| [Gather](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Gather) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gather-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gather-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gather-13) | +| [GatherElements](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GatherElements) | | +| [GatherND](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GatherND) | | +| [Gemm](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Gemm) | [7-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gemm-7), [9-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gemm-9), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gemm-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Gemm-13) | +| [GlobalAveragePool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GlobalAveragePool) | [1+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#GlobalAveragePool-1) | +| [GlobalLpPool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GlobalLpPool) | | +| [GlobalMaxPool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GlobalMaxPool) | [1+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#GlobalMaxPool-1) | +| [Greater](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Greater) | [7-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Greater-7), [9-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Greater-9), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Greater-13) | +| [GreaterOrEqual](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GreaterOrEqual) | | +| [GridSample](https://github.com/onnx/onnx/blob/main/docs/Operators.md#GridSample) | | +| [HammingWindow](https://github.com/onnx/onnx/blob/main/docs/Operators.md#HammingWindow) | | +| [HannWindow](https://github.com/onnx/onnx/blob/main/docs/Operators.md#HannWindow) | | +| [HardSigmoid](https://github.com/onnx/onnx/blob/main/docs/Operators.md#HardSigmoid) | | +| [HardSwish](https://github.com/onnx/onnx/blob/main/docs/Operators.md#HardSwish) | | +| [Hardmax](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Hardmax) | | +| [Identity](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Identity) | [1-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Identity-1), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Identity-13), [14-15](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Identity-14), [16+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Identity-16) | +| [If](https://github.com/onnx/onnx/blob/main/docs/Operators.md#If) | | +| [InstanceNormalization](https://github.com/onnx/onnx/blob/main/docs/Operators.md#InstanceNormalization) | [6+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#InstanceNormalization-6) | +| [IsInf](https://github.com/onnx/onnx/blob/main/docs/Operators.md#IsInf) | | +| [IsNaN](https://github.com/onnx/onnx/blob/main/docs/Operators.md#IsNaN) | | +| [LRN](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LRN) | | +| [LSTM](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LSTM) | | +| [LayerNormalization](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LayerNormalization) | | +| [LeakyRelu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LeakyRelu) | [6-15](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#LeakyRelu-6), [16+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#LeakyRelu-16) | +| [Less](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Less) | [7-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-7), [9-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-9), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Less-13) | +| [LessOrEqual](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LessOrEqual) | | +| [Log](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Log) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Log-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Log-13) | +| [LogSoftmax](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LogSoftmax) | | +| [Loop](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Loop) | | +| [LpNormalization](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LpNormalization) | | +| [LpPool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#LpPool) | | +| [MatMul](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MatMul) | [1-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MatMul-1), [9-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MatMul-9), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MatMul-13) | +| [MatMulInteger](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MatMulInteger) | | +| [Max](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Max) | | +| [MaxPool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MaxPool) | [1-7](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MaxPool-1), [8-9](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MaxPool-8), [10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MaxPool-10), [11](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MaxPool-11), [12+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#MaxPool-12) | +| [MaxRoiPool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MaxRoiPool) | | +| [MaxUnpool](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MaxUnpool) | | +| [Mean](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Mean) | | +| [MeanVarianceNormalization](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MeanVarianceNormalization) | | +| [MelWeightMatrix](https://github.com/onnx/onnx/blob/main/docs/Operators.md#MelWeightMatrix) | | +| [Min](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Min) | | +| [Mod](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Mod) | | +| [Mul](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Mul) | [7-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Mul-7), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Mul-13), [14+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Mul-14) | +| [Multinomial](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Multinomial) | | +| [Neg](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Neg) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Neg-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Neg-13) | +| [NegativeLogLikelihoodLoss](https://github.com/onnx/onnx/blob/main/docs/Operators.md#NegativeLogLikelihoodLoss) | | +| [NonMaxSuppression](https://github.com/onnx/onnx/blob/main/docs/Operators.md#NonMaxSuppression) | | +| [NonZero](https://github.com/onnx/onnx/blob/main/docs/Operators.md#NonZero) | | +| [Not](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Not) | [1+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Not-1) | +| [OneHot](https://github.com/onnx/onnx/blob/main/docs/Operators.md#OneHot) | | +| [Optional](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Optional) | | +| [OptionalGetElement](https://github.com/onnx/onnx/blob/main/docs/Operators.md#OptionalGetElement) | | +| [OptionalHasElement](https://github.com/onnx/onnx/blob/main/docs/Operators.md#OptionalHasElement) | | +| [Or](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Or) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Or-7) | +| [PRelu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#PRelu) | [7-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#PRelu-7), [9-15](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#PRelu-9), [16+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#PRelu-16) | +| [Pad](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Pad) | [2-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pad-2), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pad-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pad-13) | +| [Pow](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Pow) | [7-11](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pow-7), [12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pow-12), [13-14](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pow-13), [15+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Pow-15) | +| [QLinearConv](https://github.com/onnx/onnx/blob/main/docs/Operators.md#QLinearConv) | | +| [QLinearMatMul](https://github.com/onnx/onnx/blob/main/docs/Operators.md#QLinearMatMul) | | +| [QuantizeLinear](https://github.com/onnx/onnx/blob/main/docs/Operators.md#QuantizeLinear) | | +| [RNN](https://github.com/onnx/onnx/blob/main/docs/Operators.md#RNN) | | +| [RandomNormal](https://github.com/onnx/onnx/blob/main/docs/Operators.md#RandomNormal) | | +| [RandomNormalLike](https://github.com/onnx/onnx/blob/main/docs/Operators.md#RandomNormalLike) | | +| [RandomUniform](https://github.com/onnx/onnx/blob/main/docs/Operators.md#RandomUniform) | | +| [RandomUniformLike](https://github.com/onnx/onnx/blob/main/docs/Operators.md#RandomUniformLike) | | +| [Range](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Range) | | +| [Reciprocal](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Reciprocal) | | +| [ReduceL1](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceL1) | | +| [ReduceL2](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceL2) | | +| [ReduceLogSum](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceLogSum) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceLogSum-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceLogSum-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceLogSum-13) | +| [ReduceLogSumExp](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceLogSumExp) | | +| [ReduceMax](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceMax) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMax-1), [11](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMax-11), [12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMax-12), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMax-13) | +| [ReduceMean](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceMean) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMean-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMean-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMean-13) | +| [ReduceMin](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceMin) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMin-1), [11](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMin-11), [12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMin-12), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceMin-13) | +| [ReduceProd](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceProd) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceProd-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceProd-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceProd-13) | +| [ReduceSum](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceSum) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceSum-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceSum-11) | +| [ReduceSumSquare](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReduceSumSquare) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceSumSquare-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceSumSquare-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#ReduceSumSquare-13) | +| [Relu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Relu) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Relu-6), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Relu-13), [14+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Relu-14) | +| [Reshape](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Reshape) | [5-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Reshape-5), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Reshape-13), [14+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Reshape-14) | +| [Resize](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Resize) | [10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Resize-10), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Resize-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Resize-13) | +| [ReverseSequence](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ReverseSequence) | | +| [RoiAlign](https://github.com/onnx/onnx/blob/main/docs/Operators.md#RoiAlign) | | +| [Round](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Round) | | +| [STFT](https://github.com/onnx/onnx/blob/main/docs/Operators.md#STFT) | | +| [Scan](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Scan) | | +| [Scatter](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Scatter) | | +| [ScatterElements](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ScatterElements) | | +| [ScatterND](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ScatterND) | | +| [Selu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Selu) | | +| [SequenceAt](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceAt) | | +| [SequenceConstruct](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceConstruct) | | +| [SequenceEmpty](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceEmpty) | | +| [SequenceErase](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceErase) | | +| [SequenceInsert](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceInsert) | | +| [SequenceLength](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceLength) | | +| [SequenceMap](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SequenceMap) | | +| [Shape](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Shape) | [1-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Shape-1), [13-14](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Shape-13), [15+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Shape-15) | +| [Shrink](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Shrink) | | +| [Sigmoid](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sigmoid) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sigmoid-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sigmoid-13) | +| [Sign](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sign) | | +| [Sin](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sin) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sin-7) | +| [Sinh](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sinh) | | +| [Size](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Size) | | +| [Slice](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Slice) | [1-9](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Slice-1), [10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Slice-10), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Slice-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Slice-13) | +| [Softmax](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Softmax) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Softmax-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Softmax-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Softmax-13) | +| [SoftmaxCrossEntropyLoss](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SoftmaxCrossEntropyLoss) | | +| [Softplus](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Softplus) | | +| [Softsign](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Softsign) | | +| [SpaceToDepth](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SpaceToDepth) | | +| [Split](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Split) | [2-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Split-2), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Split-11) | +| [SplitToSequence](https://github.com/onnx/onnx/blob/main/docs/Operators.md#SplitToSequence) | | +| [Sqrt](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sqrt) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sqrt-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sqrt-13) | +| [Squeeze](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Squeeze) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Squeeze-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Squeeze-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Squeeze-13) | +| [StringNormalizer](https://github.com/onnx/onnx/blob/main/docs/Operators.md#StringNormalizer) | | +| [Sub](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sub) | [7-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sub-7), [13](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sub-13), [14+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sub-14) | +| [Sum](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Sum) | [6-7](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sum-6), [8-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sum-8), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Sum-13) | +| [Tan](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Tan) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Tan-7) | +| [Tanh](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Tanh) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Tanh-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Tanh-13) | +| [TfIdfVectorizer](https://github.com/onnx/onnx/blob/main/docs/Operators.md#TfIdfVectorizer) | | +| [ThresholdedRelu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#ThresholdedRelu) | | +| [Tile](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Tile) | [6-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Tile-6), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Tile-13) | +| [TopK](https://github.com/onnx/onnx/blob/main/docs/Operators.md#TopK) | | +| [Transpose](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Transpose) | [1-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Transpose-1), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Transpose-13) | +| [Trilu](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Trilu) | | +| [Unique](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Unique) | | +| [Unsqueeze](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Unsqueeze) | [1-10](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Unsqueeze-1), [11-12](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Unsqueeze-11), [13+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Unsqueeze-13) | +| [Upsample](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Upsample) | [7-8](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Upsample-7), [9](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Upsample-9) | +| [Where](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Where) | | +| [Xor](https://github.com/onnx/onnx/blob/main/docs/Operators.md#Xor) | [7+](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Xor-7) | diff --git a/js/web/lib/onnxjs/backends/webgl/ops/conv-transpose.ts b/js/web/lib/onnxjs/backends/webgl/ops/conv-transpose.ts index ffbe0f7870..0da1d64871 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/conv-transpose.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/conv-transpose.ts @@ -91,7 +91,7 @@ const createUnpackedConvTransposeProgramInfo = ivec4 coords = getOutputCoords(); int batch = coords.x; int output_channel = coords.y; - + ivec2 loc = coords.zw + pads; int group_id = output_channel / ${outputChannelsPerGroup}; @@ -192,7 +192,7 @@ export const parseConvTransposeAttributes: OperatorInitialization { // Refer to the below link for all input checks - // https://github.com/onnx/onnx/blob/master/docs/Operators.md#Conv + // https://github.com/onnx/onnx/blob/main/docs/Operators.md#Conv if (!inputs || (inputs.length !== 2 && inputs.length !== 3)) { throw new Error('Conv requires 2 or 3 inputs'); } diff --git a/js/web/lib/onnxjs/backends/webgl/ops/conv.ts b/js/web/lib/onnxjs/backends/webgl/ops/conv.ts index b6527bf2c4..ea623f5c4d 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/conv.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/conv.ts @@ -129,7 +129,7 @@ export const parseConvAttributes: OperatorInitialization = (node const validateInputs = (inputs: Tensor[], attributes: ConvAttributes): void => { // Refer to the below link for all input checks - // https://github.com/onnx/onnx/blob/master/docs/Operators.md#Conv + // https://github.com/onnx/onnx/blob/main/docs/Operators.md#Conv if (!inputs || (inputs.length !== 2 && inputs.length !== 3)) { throw new Error('Conv requires 2 or 3 inputs'); } diff --git a/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts b/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts index a9a81fcd48..028ef3b2a1 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts @@ -117,7 +117,8 @@ const createPackedResizeProgramInfo = const unpackChannel = unpackFromChannel(); const shaderSource = ` const vec2 inputWH = vec2(${inputHeight}.0, ${inputWidth}.0); - const vec4 scaleWHWH = vec4(${scalesHeight}.0, ${scalesWidth}.0, ${scalesHeight}.0, ${scalesWidth}.0); + const vec4 scaleWHWH = vec4(float(${scalesHeight}), float(${scalesWidth}), float(${scalesHeight}), float(${ + scalesWidth})); ${unpackChannel} ${getSourceFracIndex} float getAValue(int x10, int r, int c, int d) { @@ -263,4 +264,4 @@ const parseScalesDataFromOutputSize = // roi = new Array(inputs[0].dims.length * 2).fill(0); // } // return roi; -// }; \ No newline at end of file +// }; diff --git a/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts b/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts index 8e2356c8e2..585fbf7bbf 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts @@ -54,8 +54,8 @@ export const parseSoftmaxAttributesV13: OperatorInitialization createAttributeWithCacheKey({axis: node.attributes.getInt('axis', -1)}); // The "semantic" meaning of axis has changed in opset-13. -// Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax -// with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations +// Please compare: https://github.com/onnx/onnx/blob/main/docs/Operators.md#Softmax +// with https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Softmax-11 for detailed explanations // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already // the innermost dim @@ -279,4 +279,4 @@ const validateInputs = (inputs: Tensor[]): void => { if (inputs[0].type !== 'float32' && inputs[0].type !== 'float64') { throw new Error('Invalid input type'); } -}; \ No newline at end of file +}; diff --git a/js/web/lib/onnxjs/util.ts b/js/web/lib/onnxjs/util.ts index c791493d6e..54806f90f2 100644 --- a/js/web/lib/onnxjs/util.ts +++ b/js/web/lib/onnxjs/util.ts @@ -624,7 +624,7 @@ export class ShapeUtil { * originalDims = [2,2] and shapeHints = [0,-1] will return [2,2] * originalDims = [2,2] and shapeHints = [4] will return [4] * originalDims = [2,2] and shapeHints = [5] will throw an exception - * https://github.com/onnx/onnx/blob/master/docs/Operators.md#Reshape + * https://github.com/onnx/onnx/blob/main/docs/Operators.md#Reshape */ static calculateReshapedDims(originalDims: readonly number[], shapeHints: ArrayLike): number[] { diff --git a/js/web/package-lock.json b/js/web/package-lock.json index 7a4a42ed77..e1cc9c00b0 100644 --- a/js/web/package-lock.json +++ b/js/web/package-lock.json @@ -25,7 +25,7 @@ "@types/karma": "^6.1.0", "@types/long": "^4.0.1", "@types/minimatch": "^3.0.4", - "@types/minimist": "^1.2.1", + "@types/minimist": "^1.2.2", "@types/mocha": "^8.2.2", "@types/npmlog": "^4.1.2", "@types/platform": "^1.3.3", @@ -48,7 +48,7 @@ "karma-safari-applescript-launcher": "^0.1.0", "karma-sourcemap-loader": "^0.3.8", "minimatch": "^3.0.4", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mocha": "^9.2.1", "node-polyfill-webpack-plugin": "^1.1.0", "npmlog": "^4.1.2", @@ -426,9 +426,9 @@ "dev": true }, "node_modules/@types/minimist": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.1.tgz", - "integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", "dev": true }, "node_modules/@types/mocha": { @@ -7567,9 +7567,9 @@ "dev": true }, "@types/minimist": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.1.tgz", - "integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.2.tgz", + "integrity": "sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==", "dev": true }, "@types/mocha": { diff --git a/js/web/package.json b/js/web/package.json index 6d45a010de..5c3e607afc 100644 --- a/js/web/package.json +++ b/js/web/package.json @@ -12,12 +12,12 @@ "version": "1.13.0", "jsdelivr": "dist/ort.min.js", "dependencies": { - "onnx-proto": "^4.0.4", "flatbuffers": "^1.12.0", - "long": "^4.0.0", - "platform": "^1.3.6", "guid-typescript": "^1.0.9", - "onnxruntime-common": "file:../common" + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "file:../common", + "platform": "^1.3.6" }, "scripts": { "prepare": "tsc", @@ -42,7 +42,7 @@ "@types/karma": "^6.1.0", "@types/long": "^4.0.1", "@types/minimatch": "^3.0.4", - "@types/minimist": "^1.2.1", + "@types/minimist": "^1.2.2", "@types/mocha": "^8.2.2", "@types/npmlog": "^4.1.2", "@types/platform": "^1.3.3", @@ -65,7 +65,7 @@ "karma-safari-applescript-launcher": "^0.1.0", "karma-sourcemap-loader": "^0.3.8", "minimatch": "^3.0.4", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mocha": "^9.2.1", "node-polyfill-webpack-plugin": "^1.1.0", "npmlog": "^4.1.2", diff --git a/js/web/script/generate-operator-md.ts b/js/web/script/generate-operator-md.ts index f6bb4bac58..0a1d845faf 100644 --- a/js/web/script/generate-operator-md.ts +++ b/js/web/script/generate-operator-md.ts @@ -27,7 +27,7 @@ function formatDesc(opType: string, range: [number, number], support: boolean, l let versionDesc = ''; if (support) { versionDesc = last ? `${range[0]}+` : range[0] === range[1] ? `${range[0]}` : `${range[0]}-${range[1]}`; - versionDesc = `[${versionDesc}](https://github.com/onnx/onnx/blob/master/docs/Changelog.md#${opType}-${range[0]})`; + versionDesc = `[${versionDesc}](https://github.com/onnx/onnx/blob/main/docs/Changelog.md#${opType}-${range[0]})`; } return versionDesc; } @@ -73,7 +73,7 @@ const opTypes = Array.from(onnxOpset.keys()).sort(); const doc = fs.createWriteStream(path.join(__dirname, '../docs/operators.md')); doc.write(`## Operators Support Table${EOL}${EOL}`); -doc.write(`The following table shows [ai.onnx](https://github.com/onnx/onnx/blob/master/docs/Operators.md)\ +doc.write(`The following table shows [ai.onnx](https://github.com/onnx/onnx/blob/main/docs/Operators.md)\ operators from which onnx opset version are currently supported by ONNX Runtime Web. For example, \`4-6, 8+\` means\ ONNX Runtime Web currently support opset version 4 to 6, 8 and above.${EOL}${EOL}`); doc.write(`See [Compatibility](../README.md#Compatibility) for a list of the supported platforms.${EOL}${EOL}`); @@ -99,7 +99,7 @@ for (const type of opTypes) { webgl.push(formatDesc(type, versionRange, checkSupport(type, versionRange, webglCheckOnlyRules), last)); } - doc.write(`| [${type}](https://github.com/onnx/onnx/blob/master/docs/Operators.md#${type}) | ${ + doc.write(`| [${type}](https://github.com/onnx/onnx/blob/main/docs/Operators.md#${type}) | ${ webgl.filter(d => d.length > 0).join(', ')} |${EOL}`); } doc.end(); diff --git a/js/web/test/e2e/package.json b/js/web/test/e2e/package.json index 7e0c78e8ff..0c7855ac15 100644 --- a/js/web/test/e2e/package.json +++ b/js/web/test/e2e/package.json @@ -7,7 +7,7 @@ "karma-mocha": "^2.0.1", "karma-mocha-reporter": "^2.2.5", "light-server": "^2.9.1", - "minimist": "^1.2.5", + "minimist": "^1.2.6", "mocha": "^8.3.2" } } diff --git a/objectivec/include/ort_session.h b/objectivec/include/ort_session.h index 93c1a945ca..bac82c32d4 100644 --- a/objectivec/include/ort_session.h +++ b/objectivec/include/ort_session.h @@ -160,7 +160,7 @@ NS_ASSUME_NONNULL_BEGIN * Sets a session configuration key-value pair. * Any value for a previously set key will be overwritten. * The session configuration keys and values are documented here: - * https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h + * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h * * @param key The key. * @param value The value. @@ -212,7 +212,7 @@ NS_ASSUME_NONNULL_BEGIN * Sets a run configuration key-value pair. * Any value for a previously set key will be overwritten. * The run configuration keys and values are documented here: - * https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h + * https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/session/onnxruntime_run_options_config_keys.h * * @param key The key. * @param value The value. diff --git a/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc index f084e72dcd..a408a8f95c 100644 --- a/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc +++ b/onnxruntime/contrib_ops/cpu/attnlstm/uni_dir_attn_lstm.cc @@ -264,10 +264,6 @@ void UniDirectionalAttnLstm::Compute(const gsl::span& inputs_arg, DumpMatrix("Xt*(W[iofc]^T)", output_iofc_.data(), total_rows, hidden_size_x4); - int fused_hidden_rows = batch_size_ / hidden_num_threads_; - if (batch_size_ % hidden_num_threads_ != 0) - fused_hidden_rows++; - // NOTE: we could refine the bounds checking in the calls below that use these values to instead // explicitly check just the range for each iteration, however if it's going to run over // it should also run over on the last iteration, so this should be good enough to catch any diff --git a/onnxruntime/contrib_ops/cpu/bert/attention.cc b/onnxruntime/contrib_ops/cpu/bert/attention.cc index 1234fb019c..7af0d9d7be 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention.cc +++ b/onnxruntime/contrib_ops/cpu/bert/attention.cc @@ -117,7 +117,7 @@ Status AttentionBase::CheckInputs(const TensorShape& input_shape, } if (hidden_size % num_heads_ != 0) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "hidden_size should be divisiable by num_heads."); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "hidden_size should be divisible by num_heads."); } } else { int qkv_sizes = 0; @@ -129,12 +129,13 @@ Status AttentionBase::CheckInputs(const TensorShape& input_shape, if (qkv_hidden_sizes_[0] != qkv_hidden_sizes_[1]) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "qkv_hidden_sizes first element should be same as the second"); + "qkv_hidden_sizes first element should be same as the second"); } for (size_t i = 0; i < qkv_hidden_sizes_.size(); i++) { if (qkv_hidden_sizes_[i] % num_heads_ != 0) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "hidden_size should be divisiable by num_heads:", qkv_hidden_sizes_[i]); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "hidden_size should be divisible by num_heads:", qkv_hidden_sizes_[i]); } qkv_sizes += static_cast(qkv_hidden_sizes_[i]); @@ -164,13 +165,16 @@ Status AttentionBase::CheckInputs(const TensorShape& input_shape, return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 0 shall have length of 2"); } if (static_cast(past_dims[1]) != batch_size) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 1 shall have same length as dimension 0 of input 0"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'past' dimension 1 shall have same length as dimension 0 of input 0"); } if (static_cast(past_dims[2]) != num_heads_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 2 shall have length of num_heads", num_heads_); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'past' dimension 2 shall have length of num_heads", num_heads_); } if (static_cast(past_dims[4]) != hidden_size / num_heads_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 2 shall have length of ", hidden_size / num_heads_); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'past' dimension 2 shall have length of ", hidden_size / num_heads_); } past_sequence_length = static_cast(past_dims[3]); } @@ -179,31 +183,50 @@ Status AttentionBase::CheckInputs(const TensorShape& input_shape, const auto& mask_dims = mask_index->Shape().GetDims(); if (mask_dims.size() == 1) { if (static_cast(mask_dims[0]) != batch_size && static_cast(mask_dims[0]) != 2 * batch_size) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask_index' with 1D data shall have length of batch_size or 2 * batch_size"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'mask_index' with 1D data shall have length of batch_size or 2 * batch_size"); } } else if (mask_dims.size() == 2) { - if (static_cast(mask_dims[0]) != batch_size || static_cast(mask_dims[1]) != past_sequence_length + sequence_length) { + if (static_cast(mask_dims[0]) != batch_size || + static_cast(mask_dims[1]) != past_sequence_length + sequence_length) { // Add operator supports broadcasting. Here we handle a case with only one element in the 2nd dimension. - if ((static_cast(mask_dims[0]) == batch_size || static_cast(mask_dims[0]) == 1) && static_cast(mask_dims[1]) == 1) { - // Mask will have same value after propogation, which has same effect as no mask. + if ((static_cast(mask_dims[0]) == batch_size || static_cast(mask_dims[0]) == 1) && + static_cast(mask_dims[1]) == 1) { + // Mask will have same value after propagation, which has same effect as no mask. mask_index = nullptr; } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask_index' with 2D data shall have shape batch_size x (past_sequence_length + sequence_length)"); + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'mask_index' with 2D data shall have shape " + "batch_size x (past_sequence_length + sequence_length)"); } } } else if (mask_dims.size() == 3) { - if (static_cast(mask_dims[0]) != batch_size || mask_dims[1] != sequence_length || static_cast(mask_dims[2]) != past_sequence_length + sequence_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask_index' with 3D data shall have shape batch_size x sequence_length x (past_sequence_length + sequence_length)"); + if (static_cast(mask_dims[0]) != batch_size || + mask_dims[1] != sequence_length || + static_cast(mask_dims[2]) != past_sequence_length + sequence_length) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'mask_index' with 3D data shall have shape " + "batch_size x sequence_length x (past_sequence_length + sequence_length)"); } } else if (mask_dims.size() == 4) { - if (static_cast(mask_dims[0]) != batch_size || mask_dims[1] != 1 || mask_dims[2] != mask_dims[3] || mask_dims[2] < static_cast(past_sequence_length) + sequence_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask_index' with 4D data shall have shape batch_size x 1 x max_sequence_length x max_sequence_length)"); + if (static_cast(mask_dims[0]) != batch_size || + mask_dims[1] != 1 || + mask_dims[2] != mask_dims[3] || + mask_dims[2] < static_cast(past_sequence_length) + sequence_length) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'mask_index' with 4D data shall have shape " + "batch_size x 1 x max_sequence_length x max_sequence_length)"); } if (is_unidirectional_ == true) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask_index' with 4D data shall have is_unidirectional_ set to false"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'mask_index' with 4D data shall have is_unidirectional_ set to false"); } } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'mask_index' is expected to have 1, 2, 3 or 4 dimensions, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'mask_index' is expected to have 1, 2, 3 or 4 dimensions, got ", mask_dims.size()); } } @@ -212,24 +235,29 @@ Status AttentionBase::CheckInputs(const TensorShape& input_shape, const auto& extra_add_qk_dims = extra_add_qk->Shape().GetDims(); if (extra_add_qk_dims.size() != 4) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'extra_add_qk' is expected to have 4 dimensions, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'extra_add_qk' is expected to have 4 dimensions, got ", extra_add_qk_dims.size()); } if (extra_add_qk_dims[0] != batch_size) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'extra_add_qk' dimension 0 should be same as batch_size, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'extra_add_qk' dimension 0 should be same as batch_size, got ", extra_add_qk_dims[0]); } if (extra_add_qk_dims[1] != num_heads_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'extra_add_qk' dimension 1 should be same as number of heads, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'extra_add_qk' dimension 1 should be same as number of heads, got ", extra_add_qk_dims[1]); } if (extra_add_qk_dims[2] != sequence_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'extra_add_qk' dimension 2 should be same as sequence_length, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'extra_add_qk' dimension 2 should be same as sequence_length, got ", extra_add_qk_dims[2]); } if (extra_add_qk_dims[3] != sequence_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'extra_add_qk' dimension 3 should be same as sequence_length, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'extra_add_qk' dimension 3 should be same as sequence_length, got ", extra_add_qk_dims[3]); } } @@ -322,7 +350,6 @@ template Status Attention::PrePack(const Tensor& weights, int input_idx, AllocatorPtr alloc, /*out*/ bool& is_packed, /*out*/ PrePackedWeights* prepacked_weights) { - /* The PrePack() massages the weights to speed up Compute(), there is an option to * use shared prepacked weights in which case prepacked_weights parameter would be non-null. * @@ -375,9 +402,14 @@ Status Attention::PrePack(const Tensor& weights, int input_idx, AllocatorPtr const size_t qkv_head_size[3] = {q_hidden_size / num_heads_, k_hidden_size / num_heads_, v_hidden_size / num_heads_}; const size_t weight_matrix_col_size = q_hidden_size + k_hidden_size + v_hidden_size; - if (!IsPackWeightsSuccessful(0, alloc, qkv_head_size[0], input_hidden_size, weights_data, weight_matrix_col_size, prepacked_weights) || - !IsPackWeightsSuccessful(1, alloc, qkv_head_size[1], input_hidden_size, weights_data + (num_heads_ * qkv_head_size[0]), weight_matrix_col_size, prepacked_weights) || - !IsPackWeightsSuccessful(2, alloc, qkv_head_size[2], input_hidden_size, weights_data + (num_heads_ * (qkv_head_size[0] + qkv_head_size[1])), weight_matrix_col_size, prepacked_weights)) { + if (!IsPackWeightsSuccessful(0, alloc, qkv_head_size[0], input_hidden_size, + weights_data, weight_matrix_col_size, prepacked_weights) || + !IsPackWeightsSuccessful(1, alloc, qkv_head_size[1], input_hidden_size, + weights_data + (num_heads_ * qkv_head_size[0]), + weight_matrix_col_size, prepacked_weights) || + !IsPackWeightsSuccessful(2, alloc, qkv_head_size[2], input_hidden_size, + weights_data + (num_heads_ * (qkv_head_size[0] + qkv_head_size[1])), + weight_matrix_col_size, prepacked_weights)) { if (prepacked_weights == nullptr) { FreePackedWeights(packed_weights_, qkv_hidden_sizes_.size()); } @@ -469,7 +501,8 @@ Status Attention::Compute(OpKernelContext* context) const { // gemm_data(BS, NT) = input(BS, D) x weights(D, NT) + bias(NT) // D (input_hidden_size) is hidden dimension of input, where D could be larger than any of the hidden_sizes // (NH) when model is pruned. T = H1 + H2 + H3, where H1, H2, H3 are head sizes of Q, K, V respectively - auto gemm_data = allocator->Alloc(SafeInt(batch_size) * sequence_length * (q_hidden_size + k_hidden_size + v_hidden_size) * element_size); + int qkv_hidden_size = (q_hidden_size + k_hidden_size + v_hidden_size); + auto gemm_data = allocator->Alloc(SafeInt(batch_size) * sequence_length * qkv_hidden_size * element_size); BufferUniquePtr gemm_buffer(gemm_data, BufferDeleter(std::move(allocator))); auto Q = reinterpret_cast(gemm_data); @@ -523,12 +556,13 @@ Status Attention::Compute(OpKernelContext* context) const { // C: QKV[qkv_index] (BxNxSxT) (B.N.)S x T S x H if (is_prepack_) { uint8_t* packed_weight; - packed_weight = static_cast(packed_weights_[qkv_index].get()) + packed_weights_size_[qkv_index] * (weights_offset / head_size); + packed_weight = static_cast(packed_weights_[qkv_index].get()) + + packed_weights_size_[qkv_index] * (weights_offset / head_size); MlasGemm( CblasNoTrans, // TransA = no sequence_length, // M = S - head_size, // N = H + head_size, // N = H input_hidden_size, // K = D 1.0f, // alpha input_data + input_offset, // A @@ -540,20 +574,20 @@ Status Attention::Compute(OpKernelContext* context) const { nullptr); // use single-thread } else { math::GemmEx( - CblasNoTrans, // TransA = no - CblasNoTrans, // TransB = no - sequence_length, // M = S - head_size, // N = H - input_hidden_size, // K = D - 1.0f, // alpha - input_data + input_offset, // A - input_hidden_size, // lda = D - weights_data + weights_offset, // B - q_hidden_size + k_hidden_size + v_hidden_size,// ldb = NH1 + NH2 + NH3 - 1.0f, // beta - qkv_dest + qkv_offset, // C - head_size, // ldc - nullptr // use single-thread + CblasNoTrans, // TransA = no + CblasNoTrans, // TransB = no + sequence_length, // M = S + head_size, // N = H + input_hidden_size, // K = D + 1.0f, // alpha + input_data + input_offset, // A + input_hidden_size, // lda = D + weights_data + weights_offset, // B + q_hidden_size + k_hidden_size + v_hidden_size, // ldb = NH1 + NH2 + NH3 + 1.0f, // beta + qkv_dest + qkv_offset, // C + head_size, // ldc + nullptr // use single-thread ); } } diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_base.h index 0855cb3377..feee7e9d6d 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention_base.h +++ b/onnxruntime/contrib_ops/cpu/bert/attention_base.h @@ -3,6 +3,7 @@ #pragma once +#include #include "core/common/common.h" #include "core/framework/op_kernel.h" @@ -17,7 +18,7 @@ class AttentionBase { const TensorShape& bias_shape, const Tensor*& mask_index, // For dummy mask with shape (1, 1) or (batch_size, 1), it will be updated to nullptr. const Tensor* past, - const Tensor *extra_add_qk, + const Tensor* extra_add_qk, const int max_threads_per_block) const; Tensor* GetPresent(OpKernelContext* context, @@ -45,11 +46,11 @@ class AttentionBase { const TensorShape& bias_shape, const Tensor*& mask_index, // For dummy mask with shape (1, 1) or (batch_size, 1), it will be updated to nullptr. const Tensor* past, - const Tensor *extra_add_qk) const; + const Tensor* extra_add_qk) const; - int num_heads_; // number of attention heads - bool is_unidirectional_; // whether every token can only attend to previous tokens. - std::vector qkv_hidden_sizes_; // Q, K, V path hidden layer sizes + int num_heads_; // number of attention heads + bool is_unidirectional_; // whether every token can only attend to previous tokens. + std::vector qkv_hidden_sizes_; // Q, K, V path hidden layer sizes }; } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h index 69e2017601..1d08638a7c 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h +++ b/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h @@ -9,12 +9,7 @@ #include "core/common/common.h" #include "core/common/safeint.h" #include "core/framework/op_kernel.h" -//TODO: fix the warnings -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(push) -// Chance of arithmetic overflow could be reduced -#pragma warning(disable : 26451) -#endif + namespace onnxruntime { namespace contrib { @@ -51,8 +46,8 @@ class AttentionCPUBase : public AttentionBase { // I. attention_probs(B, N, S, S*) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, S*, H -> B, N, H, S*) + // 1 x mask_data(B, N, S, S*) // II.attention_probs(B, N, S, S*) = Softmax(attention_probs) - size_t attention_probs_bytes = SafeInt(batch_size) * num_heads_ * sequence_length * all_sequence_length * sizeof(T); - auto attention_probs = allocator->Alloc(attention_probs_bytes); + size_t bytes = SafeInt(batch_size) * num_heads_ * sequence_length * all_sequence_length * sizeof(T); + auto attention_probs = allocator->Alloc(bytes); BufferUniquePtr scratch_buffer(attention_probs, BufferDeleter(allocator)); bool has_unidirectional = (is_unidirectional_ && sequence_length > 1); @@ -66,7 +61,9 @@ class AttentionCPUBase : public AttentionBase { BufferUniquePtr mask_data_buffer(mask_data, BufferDeleter(allocator)); const int32_t* mask_index_data = mask_index != nullptr ? mask_index->Data() : nullptr; - gsl::span mask_index_dims = mask_index != nullptr ? mask_index->Shape().GetDims() : gsl::span{}; + gsl::span mask_index_dims = mask_index != nullptr + ? mask_index->Shape().GetDims() + : gsl::span{}; const T* past_data = past != nullptr ? past->Data() : nullptr; T* present_data = present != nullptr ? present->MutableData() : nullptr; @@ -77,7 +74,8 @@ class AttentionCPUBase : public AttentionBase { ComputeAttentionProbs(static_cast(attention_probs), Q, K, mask_index_data, mask_index_dims, static_cast(mask_data), has_unidirectional, - batch_size, sequence_length, past_sequence_length, qk_head_size == 0 ? v_head_size : qk_head_size, + batch_size, sequence_length, past_sequence_length, + qk_head_size == 0 ? v_head_size : qk_head_size, past_data, present_data, tp, extra_add_qk_data); // Compute the attentionScore * Value. It does: out_tmp(B, N, S, H) = attention_probs(B, N, S, S*) x V(B, N, S*, H) @@ -85,7 +83,8 @@ class AttentionCPUBase : public AttentionBase { allocator->Alloc(SafeInt(batch_size) * num_heads_ * sequence_length * v_head_size * sizeof(T)); BufferUniquePtr out_tmp_buffer(out_tmp_data, BufferDeleter(std::move(allocator))); - ComputeVxAttentionScore(output->MutableData(), static_cast(out_tmp_data), static_cast(attention_probs), V, + ComputeVxAttentionScore(output->MutableData(), static_cast(out_tmp_data), + static_cast(attention_probs), V, batch_size, sequence_length, past_sequence_length, v_head_size, v_hidden_size, past_data, present_data, tp); @@ -98,21 +97,21 @@ class AttentionCPUBase : public AttentionBase { // 1 x mask_data(B, N, S, S*) // II.attention_probs(B, N, S, S*) = Softmax(attention_probs) template - void ComputeAttentionProbs(T* attention_probs, // output buffer for the attention probs. Its size is BxNxSxS* - const T* Q, // Q data. Its size is BxNxSxH - const T* K, // k data. Its size is BxNxSxH - const int32_t* mask_index, // mask index. nullptr if no mask or its size is B - gsl::span mask_index_dims, // mask index shape - T* mask_data, // buffer for mask data. It is nullptr if mask_index is nullptr and not unidirectional, otherwise its shape is BxSxS* - bool has_unidirectional, // has unidirectional mask - int batch_size, // batch size of self-attention - int sequence_length, // sequence length of self-attention - int past_sequence_length, // sequence length of past state - int head_size, // head size of self-attention - const T* past, // past state - T* present, // present state - ThreadPool* tp, // thread pool - const T* extra_add_qk_data // extra add matrix with shape BxNxSxS* + void ComputeAttentionProbs(T* attention_probs, // output buffer with size BxNxSxS* + const T* Q, // Q data. Its size is BxNxSxH + const T* K, // k data. Its size is BxNxSxH + const int32_t* mask_index, // mask index. nullptr if no mask. + gsl::span mask_index_dims, // mask index shape + T* mask_data, // buffer for mask data. + bool has_unidirectional, // has unidirectional mask + int batch_size, // batch size of self-attention + int sequence_length, // sequence length of self-attention + int past_sequence_length, // sequence length of past state + int head_size, // head size of self-attention + const T* past, // past state + T* present, // present state + ThreadPool* tp, // thread pool + const T* extra_add_qk_data // extra add matrix with shape BxNxSxS* ) const { const int all_sequence_length = past_sequence_length + sequence_length; // S* = S' + S const size_t past_chunk_length = static_cast(past_sequence_length) * head_size; // S' x H @@ -120,10 +119,13 @@ class AttentionCPUBase : public AttentionBase { const size_t present_chunk_length = past_chunk_length + input_chunk_length; // S* x H { + // mask_data is nullptr when mask_index is nullptr and not unidirectional, otherwise its shape is BxSxS* if (mask_data != nullptr) { - PrepareMask(mask_index, mask_index_dims, mask_data, has_unidirectional, batch_size, sequence_length, past_sequence_length); + PrepareMask(mask_index, mask_index_dims, mask_data, + has_unidirectional, batch_size, sequence_length, past_sequence_length); } else { // no any mask - memset(attention_probs, 0, static_cast(batch_size) * num_heads_ * sequence_length * all_sequence_length * sizeof(T)); + size_t bytes = static_cast(batch_size) * num_heads_ * sequence_length * all_sequence_length * sizeof(T); + memset(attention_probs, 0, bytes); } const int loop_len = batch_size * num_heads_; @@ -247,6 +249,3 @@ class AttentionCPUBase : public AttentionBase { } // namespace contrib } // namespace onnxruntime -#if defined(_MSC_VER) && !defined(__clang__) -#pragma warning(pop) -#endif diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_helper.h b/onnxruntime/contrib_ops/cpu/bert/attention_helper.h index 6e21d82d6a..9b05f30ce8 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention_helper.h +++ b/onnxruntime/contrib_ops/cpu/bert/attention_helper.h @@ -99,7 +99,9 @@ void PrepareMask(const int32_t* mask_index, } bool is_raw_attention_mask = (nullptr != mask_index && mask_index_dims.size() == 2); - bool has_mask_start_position = (nullptr != mask_index && mask_index_dims.size() == 1 && static_cast(mask_index_dims.at(0)) == 2 * batch_size); + bool has_mask_start_position = (nullptr != mask_index && + mask_index_dims.size() == 1 && + static_cast(mask_index_dims.at(0)) == 2 * batch_size); for (int b_i = 0; b_i < batch_size; b_i++) { // TODO: mask_index can be used in softmax to save some calculation. @@ -150,7 +152,12 @@ void PrepareMask(const int32_t* mask_index, // Concatenate a past state chunk S'xH with input state chunk SxH into present state chunk S*xH // Returns a pointer to the start of present state chunk. template -T* ConcatStateChunk(const T* past, const T* chunk, T* present, size_t past_chunk_length, size_t present_chunk_length, std::ptrdiff_t i) { +T* ConcatStateChunk(const T* past, + const T* chunk, + T* present, + size_t past_chunk_length, + size_t present_chunk_length, + std::ptrdiff_t i) { T* start = present + i * present_chunk_length; T* p = start; diff --git a/onnxruntime/contrib_ops/cpu/bert/bias_gelu_helper.h b/onnxruntime/contrib_ops/cpu/bert/bias_gelu_helper.h index 2500dd0f35..86552f8888 100644 --- a/onnxruntime/contrib_ops/cpu/bert/bias_gelu_helper.h +++ b/onnxruntime/contrib_ops/cpu/bert/bias_gelu_helper.h @@ -12,6 +12,6 @@ namespace bias_gelu_helper { Status CheckInputs(const OpKernelContext* context); -} +} // namespace bias_gelu_helper } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm.cc b/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm.cc index 44b972e897..fdc5573361 100644 --- a/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm.cc +++ b/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm.cc @@ -38,7 +38,6 @@ EmbedLayerNorm::EmbedLayerNorm(const OpKernelInfo& op_kernel_info) : EmbedLayerNormBase(op_kernel_info) { } - template Status EmbedLayerNorm::Compute(OpKernelContext* context) const { ORT_RETURN_IF_ERROR(embed_layer_norm::CheckInputs(context)); @@ -49,8 +48,8 @@ Status EmbedLayerNorm::Compute(OpKernelContext* context) const { const Tensor* segment_embedding = context->Input(4); // optional. nullptr if it's distill-bert const Tensor* gamma = context->Input(5); const Tensor* beta = context->Input(6); - const Tensor* mask = context->Input(7); // optional. nullptr if not provided - const Tensor* position_ids = context->Input(8); // optional. nullptr if not provided + const Tensor* mask = context->Input(7); // optional. nullptr if not provided + const Tensor* position_ids = context->Input(8); // optional. nullptr if not provided const auto& input_dims = input_ids->Shape().GetDims(); int64_t hidden_size = word_embedding->Shape()[1]; @@ -86,58 +85,62 @@ Status EmbedLayerNorm::Compute(OpKernelContext* context) const { std::atomic_bool failed{false}; int n = batch_size * sequence_length; - concurrency::ThreadPool::TryBatchParallelFor(context->GetOperatorThreadPool(), n, [=, &failed](ptrdiff_t index) { - int word_col_index = input_ids_data[index]; - if (word_col_index < 0 || word_col_index >= word_embedding_length) { - failed.store(true, std::memory_order_release); - return; - } - int position_col_index = (position_ids_data == nullptr) ? index % sequence_length : position_ids_data[index]; - if (position_col_index >= position_embedding_length) { - failed.store(true, std::memory_order_release); - return; - } - int segment_col_index = 0; - if (nullptr != segment_ids_data) { - segment_col_index = segment_ids_data[index]; - if (segment_col_index < 0 || segment_col_index >= segment_embedding_length) { - failed.store(true, std::memory_order_release); - return; - } - } + concurrency::ThreadPool::TryBatchParallelFor( + context->GetOperatorThreadPool(), n, [=, &failed](ptrdiff_t index) { + int word_col_index = input_ids_data[index]; + if (word_col_index < 0 || word_col_index >= word_embedding_length) { + failed.store(true, std::memory_order_release); + return; + } + int position_col_index = (position_ids_data == nullptr) ? index % sequence_length : position_ids_data[index]; + if (position_col_index >= position_embedding_length) { + failed.store(true, std::memory_order_release); + return; + } + int segment_col_index = 0; + if (nullptr != segment_ids_data) { + segment_col_index = segment_ids_data[index]; + if (segment_col_index < 0 || segment_col_index >= segment_embedding_length) { + failed.store(true, std::memory_order_release); + return; + } + } - T* y = output_data + index * hidden_size; - T* y1 = nullptr; - if (embedding_sum_data != nullptr) { - y1 = embedding_sum_data + index * hidden_size; - } - const T* input_word_embedding = word_embedding_data + word_col_index * hidden_size; - const T* input_position_embedding = position_embedding_data + position_col_index * hidden_size; - const T* input_segment_embedding = (nullptr == segment_embedding_data) ? nullptr : segment_embedding_data + segment_col_index * hidden_size; + T* y = output_data + index * hidden_size; + T* y1 = nullptr; + if (embedding_sum_data != nullptr) { + y1 = embedding_sum_data + index * hidden_size; + } + const T* input_word_embedding = word_embedding_data + word_col_index * hidden_size; + const T* input_position_embedding = position_embedding_data + position_col_index * hidden_size; + const T* input_segment_embedding = (nullptr == segment_embedding_data) + ? nullptr + : segment_embedding_data + segment_col_index * hidden_size; - T sum = static_cast(0); - for (int i = 0; i < hidden_size; i++) { - T subtotal = input_word_embedding[i] + input_position_embedding[i]; - if (nullptr != segment_embedding_data) - subtotal += input_segment_embedding[i]; - y[i] = subtotal; - if (y1 != nullptr) { - y1[i] = subtotal; - } - sum += subtotal; - } - T mean = sum / hidden_size; - sum = 0; - for (int i = 0; i < hidden_size; i++) { - T a = y[i] - mean; - y[i] = a; - sum += a * a; - } - T e = sqrt(sum / hidden_size + static_cast(epsilon())); - for (int i = 0; i < hidden_size; i++) { - y[i] = y[i] / e * gamma_data[i] + beta_data[i]; - } - }, 0); + T sum = static_cast(0); + for (int i = 0; i < hidden_size; i++) { + T subtotal = input_word_embedding[i] + input_position_embedding[i]; + if (nullptr != segment_embedding_data) + subtotal += input_segment_embedding[i]; + y[i] = subtotal; + if (y1 != nullptr) { + y1[i] = subtotal; + } + sum += subtotal; + } + T mean = sum / hidden_size; + sum = 0; + for (int i = 0; i < hidden_size; i++) { + T a = y[i] - mean; + y[i] = a; + sum += a * a; + } + T e = sqrt(sum / hidden_size + static_cast(epsilon())); + for (int i = 0; i < hidden_size; i++) { + y[i] = y[i] / e * gamma_data[i] + beta_data[i]; + } + }, + 0); if (failed.load(std::memory_order_acquire)) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "input index out of range"); diff --git a/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm_helper.cc b/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm_helper.cc index e5c49a6c0d..b937912af3 100644 --- a/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm_helper.cc +++ b/onnxruntime/contrib_ops/cpu/bert/embed_layer_norm_helper.cc @@ -23,7 +23,7 @@ Status CheckInputs(const OpKernelContext* context, bool quantizedVersion) { const Tensor* mask = context->Input(7); // optional. nullptr if not provided if (!quantizedVersion) { - const Tensor* position_ids = context->Input(8); // optional. nullptr if not provided + const Tensor* position_ids = context->Input(8); // optional. nullptr if not provided if (nullptr != position_ids && input_ids->Shape() != position_ids->Shape()) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, diff --git a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc index 6a6685ac3f..97f75d297d 100644 --- a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc +++ b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.cc @@ -9,18 +9,22 @@ namespace contrib { Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, - const TensorShape& mask_shape, + const TensorShape& attention_mask_shape, const TensorShape& global_weights_shape, const TensorShape& global_bias_shape, - const TensorShape& global_shape) const { + const TensorShape& global_mask_shape) const { // Input shapes: - // input : (batch_size, sequence_length, hidden_size) - // weights : (hidden_size, 3 * hidden_size) - // bias : (3 * hidden_size) - // mask : (batch_size, sequence_length) - // global_weights : (hidden_size, 3 * hidden_size) - // global_bias : (3 * hidden_size) - // global : (batch_size, sequence_length) + // input : (batch_size, sequence_length, hidden_size) + // weights : (hidden_size, 3 * hidden_size) -- format 1 + // (3, hidden_size, hidden_size) -- format 0 + // bias : (3 * hidden_size) -- format 1 (bias for Q, K, V) + // (5 * hidden_size) -- format 0 (bias for Q, K, V, Global_K, Global_V) + // attention_mask : (batch_size, sequence_length) + // global_weights : (hidden_size, 3 * hidden_size) -- format 1 + // (3, hidden_size, hidden_size) -- format 0 + // global_bias : (3 * hidden_size) -- format 1 (bias for Global_Q, Global_K, Global_V) + // (1 * hidden_size) -- format 0 (bias for Global_Q) + // global_attention_mask : (batch_size, sequence_length) const auto& dims = input_shape.GetDims(); if (dims.size() != 3) { @@ -30,27 +34,32 @@ Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape, int batch_size = static_cast(dims[0]); int sequence_length = static_cast(dims[1]); - int hidden_size = static_cast(dims[2]); + auto hidden_size = dims[2]; if (sequence_length % (2 * window_) != 0) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Input 'input' dimension 1 should be divisiable by 2W, where W is value of the window attribute."); + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'input' dimension 1 should be divisible by 2W, where W is value of the window attribute."); } if (hidden_size % num_heads_ != 0) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Input 'input' dimension 2 should be divisiable by value of the num_heads attribute."); + "Input 'input' dimension 2 should be divisible by value of the num_heads attribute."); } const auto& weights_dims = weights_shape.GetDims(); - if (weights_dims.size() != 2) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'weights' is expected to have 2 dimensions, got ", - weights_dims.size()); - } - if (weights_dims[0] != dims[2]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Input 'weights' dimension 0 should have same length as dimension 2 of input 0"); - } - if (weights_dims[1] != 3 * weights_dims[0]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'weights' dimension 1 should be 3 times of dimension 0"); + bool use_merged_qkv_weights = (weights_shape.NumDimensions() == 2); + if (use_merged_qkv_weights) { + if (weights_dims[0] != hidden_size || weights_dims[1] != 3 * hidden_size) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'weights' shape should be (hidden_size, 3 * hidden_size) for format 1"); + } + } else { + if (weights_dims.size() != 3 || + weights_dims[0] != 3 || weights_dims[1] != hidden_size || weights_dims[2] != hidden_size) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'weights' shape should be (3, hidden_size, hidden_size) for format 0"); + } } const auto& bias_dims = bias_shape.GetDims(); @@ -58,32 +67,45 @@ Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape, return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'bias' is expected to have 1 dimension, got ", bias_dims.size()); } - if (bias_dims[0] != weights_dims[1]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Input 'bias' dimension 0 should have same length as dimension 1 of input 'weights'"); - } - const auto& mask_dims = mask_shape.GetDims(); - if (mask_dims.size() == 2) { - if (static_cast(mask_dims[0]) != batch_size || static_cast(mask_dims[1]) != sequence_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask' shall have shape batch_size x sequence_length"); + if (use_merged_qkv_weights) { + if (bias_dims[0] != 3 * hidden_size) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'bias' shape should be (3 * hidden_size) for format 1"); } } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'mask' is expected to have 2 dimensions, got ", - mask_dims.size()); + if (bias_dims[0] != 5 * hidden_size) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'bias' shape should be (5 * hidden_size) for format 0"); + } + } + + const auto& mask_dims = attention_mask_shape.GetDims(); + if (mask_dims.size() == 2) { + if (static_cast(mask_dims[0]) != batch_size || static_cast(mask_dims[1]) != sequence_length) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Inputs 'attention_mask' shape shall be (batch_size, sequence_length)"); + } + } else { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'attention_mask' is expected to have 2 dimensions, got ", mask_dims.size()); } const auto& global_weights_dims = global_weights_shape.GetDims(); - if (global_weights_dims.size() != 2) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'global_weights' is expected to have 2 dimensions, got ", - weights_dims.size()); - } - if (global_weights_dims[0] != dims[2]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Input 'global_weights' dimension 0 should have same length as dimension 2 of input 0"); - } - if (global_weights_dims[1] != 3 * global_weights_dims[0]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'global_weights' dimension 1 should be 3 times of dimension 0"); + if (use_merged_qkv_weights) { + if (global_weights_dims.size() != 2 || + global_weights_dims[0] != hidden_size || global_weights_dims[1] != 3 * hidden_size) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'global_weights' shape should be (hidden_size, 3 * hidden_size) for format 1"); + } + } else { + if (global_weights_dims.size() != 3 || global_weights_dims[0] != 3 || + global_weights_dims[1] != hidden_size || global_weights_dims[2] != hidden_size) { + return ORT_MAKE_STATUS( + ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'global_weights' shape should be (3, hidden_size, hidden_size) for format 0"); + } } const auto& global_bias_dims = global_bias_shape.GetDims(); @@ -91,19 +113,25 @@ Status LongformerAttentionBase::CheckInputs(const TensorShape& input_shape, return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'global_bias' is expected to have 1 dimension, got ", global_bias_dims.size()); } - if (global_bias_dims[0] != global_weights_dims[1]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "Input 'global_bias' dimension 0 should have same length as dimension 1 of input 'global_weights'"); - } - const auto& global_dims = global_shape.GetDims(); - if (global_dims.size() == 2) { - if (static_cast(global_dims[0]) != batch_size || static_cast(global_dims[1]) != sequence_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'global' shall have shape batch_size x sequence_length"); + if (use_merged_qkv_weights) { + if (global_bias_dims[0] != 3 * hidden_size) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'global_bias' shape should be (3 * hidden_size) for format 1"); } } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'global' is expected to have 2 dimensions, got ", - global_dims.size()); + if (global_bias_dims[0] != hidden_size) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'global_bias' shape should be (hidden_size) for format 0"); + } + } + + const auto& global_mask_dims = global_mask_shape.GetDims(); + if (global_mask_dims.size() != 2 || + static_cast(global_mask_dims[0]) != batch_size || + static_cast(global_mask_dims[1]) != sequence_length) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'global_attention_mask' shape shall be (batch_size, sequence_length)"); } return Status::OK(); diff --git a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h index 1f974db5cd..ac1cccaa83 100644 --- a/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h +++ b/onnxruntime/contrib_ops/cpu/bert/longformer_attention_base.h @@ -14,10 +14,10 @@ class LongformerAttentionBase { Status CheckInputs(const TensorShape& input_shape, const TensorShape& weights_shape, const TensorShape& bias_shape, - const TensorShape& mask_shape, + const TensorShape& attention_mask_shape, const TensorShape& global_weights_shape, const TensorShape& global_bias_shape, - const TensorShape& global_shape) const; + const TensorShape& global_attention_mask_shape) const; protected: LongformerAttentionBase(const OpKernelInfo& info) { @@ -35,8 +35,12 @@ class LongformerAttentionBase { }; namespace longformer { -// Environment variable to give a hint about choosing kernels for less memory or latency. +// Environment variable to choose compact memory kernel in experiment. Default is true. constexpr const char* kUseCompactMemory = "ORT_LONGFORMER_COMPACT_MEMORY"; + +// Environment variable to enable half4 in AddBiasTranspose kernel. Default is true. +constexpr const char* kUseHalf4 = "ORT_LONGFORMER_USE_HALF4"; + } // namespace longformer } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/bert/ngram_repeat_block.h b/onnxruntime/contrib_ops/cpu/bert/ngram_repeat_block.h index 509a35fc33..8c1f75e20a 100644 --- a/onnxruntime/contrib_ops/cpu/bert/ngram_repeat_block.h +++ b/onnxruntime/contrib_ops/cpu/bert/ngram_repeat_block.h @@ -68,16 +68,16 @@ class NGramRepeatBlock : public OpKernel { concurrency::ThreadPool* tp = context->GetOperatorThreadPool(); concurrency::ThreadPool::TryParallelFor( - tp, batch_size, static_cast(cur_len * ngram_size_), - [&lambda](ptrdiff_t first, ptrdiff_t last) { - for (auto b = static_cast(first), end = static_cast(last); b < end; ++b) { - lambda(b); - } - } - ); + tp, batch_size, static_cast(cur_len * ngram_size_), + [&lambda](ptrdiff_t first, ptrdiff_t last) { + for (auto b = static_cast(first), end = static_cast(last); b < end; ++b) { + lambda(b); + } + }); return Status::OK(); } + private: int64_t ngram_size_; }; diff --git a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc index 89cbe521a5..7816510b34 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc @@ -3,6 +3,7 @@ #include "contrib_ops/cpu/quantization/qlinear_softmax.h" +#include #include #include #include @@ -23,25 +24,30 @@ constexpr int OPSET13 = 13; namespace { -void QlinearBuildLookupTableUint32(gsl::span table, +void QlinearBuildLookupTableUint32(gsl::span table, const float x_scale, size_t reduce_len, bool is_signed) { - const double qscale = - fmin(static_cast(UINT32_MAX) / static_cast(reduce_len), static_cast(0x7fffff)); + // make sure sum(exp(x)) < max() + double bit_shift = + log(std::numeric_limits::max() / reduce_len); + double reserve_bit = std::is_same_v ? 5 : 3; + bit_shift = std::max(0.0, bit_shift - reserve_bit) / x_scale; + for (int32_t i = 0; i < 256; i++) { - double scaled_exp_xi = qscale * exp(static_cast(i - 255) * static_cast(x_scale)); - // we can't get the real max value of input tensor here, so we just assume 255. + double scaled_exp_xi = exp((i - 255 + bit_shift) * static_cast(x_scale)); + // we can't get the real max value of input tensor here, so we just assume 255-bit_shift. // in the function of `QlinearSoftmaxCPU`, - // all numbers will have a shift (255-max_value) if its max value is not 255 + // all numbers will have a shift (255-bit_shift-max_value) if its max value is not 255 // // if is_signed index = [1 2 3 ......126 127 -128 -127 ..... -3 -2 -1] // else [0 1 2 3 4 ..... 256] uint8_t index = static_cast(is_signed ? i - 128 : i); - table[index] = static_cast(lrint(scaled_exp_xi)); + table[index] = static_cast((scaled_exp_xi)); } } -void BuildLookupTableIfFixed(const OpKernelInfo& info, std::vector& fixed_lookup_table, +void BuildLookupTableIfFixed(const OpKernelInfo& info, + std::vector& fixed_lookup_table, size_t reduce_len, bool is_signed) { const Tensor* tensor_x_scale = nullptr; @@ -103,9 +109,9 @@ Status QLinearSoftmax::Compute(OpKernelContext* ctx) const { return Status::OK(); } concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); - const size_t D = opset_ < OPSET13 ? X_shape.SizeFromDimension(axis_): X_shape[axis_]; - uint32_t tmp_lookup_table[256]; - gsl::span lookup_table = GetLookupTable(ctx, tmp_lookup_table, D); + const size_t D = opset_ < OPSET13 ? X_shape.SizeFromDimension(axis_) : X_shape[axis_]; + EXP_OUT_DTYPE tmp_lookup_table[256]; + gsl::span lookup_table = GetLookupTable(ctx, tmp_lookup_table, D); if (opset_ < OPSET13) { return ComputeInternal(ctx, *X, *Y, lookup_table, axis_, thread_pool); @@ -119,8 +125,8 @@ common::Status QlinearSoftmaxCPU(size_t N, size_t D, const T* x_data, T* y_data, - const uint32_t* lookup_table, - uint32_t y_scale, + const QLinearSoftmax::EXP_OUT_DTYPE* lookup_table, + QLinearSoftmax::EXP_OUT_DTYPE y_scale, T yzp, onnxruntime::concurrency::ThreadPool* thread_pool); @@ -129,8 +135,8 @@ common::Status QlinearSoftmaxCPU(size_t N, size_t D, const uint8_t* x_data, uint8_t* y_data, - const uint32_t* lookup_table, - uint32_t y_scale, + const QLinearSoftmax::EXP_OUT_DTYPE* lookup_table, + QLinearSoftmax::EXP_OUT_DTYPE y_scale, uint8_t yzp, onnxruntime::concurrency::ThreadPool* thread_pool) { using onnxruntime::TensorOpCost; @@ -156,11 +162,11 @@ common::Status QlinearSoftmaxCPU(size_t N, // 1 3 5 ... 10 // after the shift ---> // 235 237 239 .. 255 - const uint32_t* shifted_lookuptable = lookup_table + 255 - xmax; + const QLinearSoftmax::EXP_OUT_DTYPE* shifted_lookuptable = lookup_table + 255 - xmax; size_t elements_n = D; // reduceSumUin8ToUint32: need speedup // vsum = \sum_i{e^x_i} - uint32_t vsum = 0; + QLinearSoftmax::EXP_OUT_DTYPE vsum = 0; const uint8_t* x_t_cur = x_t; do { const size_t vx = *x_t_cur++; @@ -172,12 +178,11 @@ common::Status QlinearSoftmaxCPU(size_t N, elements_n = D; x_t_cur = x_t; // elementwise div, y_i=\frac{x_i}{vsum} - const uint32_t vrounding = (vsum >> 1); do { const size_t vx = *x_t_cur++; - const uint32_t vt = shifted_lookuptable[vx]; + const QLinearSoftmax::EXP_OUT_DTYPE vt = shifted_lookuptable[vx]; // simulate round function, and re-quant to uint8 - const uint32_t vq = ((vt * c_y_scale) + vrounding) / vsum + c_y_zp; + const uint32_t vq = static_cast(std::nearbyintf(((vt * c_y_scale)) / vsum)) + c_y_zp; const uint8_t vy = vq > 255 ? static_cast(255) : static_cast(vq); *y_t++ = vy; } while (--elements_n != 0); @@ -193,8 +198,8 @@ common::Status QlinearSoftmaxCPU(size_t N, size_t D, const int8_t* x_data, int8_t* y_data, - const uint32_t* lookup_table, - uint32_t y_scale, + const QLinearSoftmax::EXP_OUT_DTYPE* lookup_table, + QLinearSoftmax::EXP_OUT_DTYPE y_scale, int8_t yzp, onnxruntime::concurrency::ThreadPool* thread_pool) { using onnxruntime::TensorOpCost; @@ -215,10 +220,10 @@ common::Status QlinearSoftmaxCPU(size_t N, // reduceMaxInt8 int8_t xmax = *std::max_element(x_t, x_t + D); const size_t adjustment = 127 - xmax; - const uint32_t* shifted_lookuptable = lookup_table; + const QLinearSoftmax::EXP_OUT_DTYPE* shifted_lookuptable = lookup_table; size_t elements_n = D; // reduceSumUin8ToUint32: need speedup - uint32_t vsum = 0; + QLinearSoftmax::EXP_OUT_DTYPE vsum = 0; const int8_t* x_t_cur = x_t; do { const size_t vx = uint8_t(adjustment + (*x_t_cur++)); @@ -230,12 +235,11 @@ common::Status QlinearSoftmaxCPU(size_t N, elements_n = D; x_t_cur = x_t; // elementwise div - const uint32_t vrounding = (vsum >> 1); do { const size_t vx = uint8_t(adjustment + (*x_t_cur++)); - const uint32_t vt = shifted_lookuptable[vx]; + const QLinearSoftmax::EXP_OUT_DTYPE vt = shifted_lookuptable[vx]; // simulate round function, and re-quant to Int8 - const uint32_t vq = ((vt * c_y_scale) + vrounding) / vsum + c_y_zp; + const int32_t vq = static_cast(std::nearbyintf(((vt * c_y_scale)) / vsum)) + c_y_zp; const int8_t vy = static_cast(vq) > 255 ? static_cast(255) : static_cast(vq); *y_t++ = vy; } while (--elements_n != 0); @@ -246,10 +250,11 @@ common::Status QlinearSoftmaxCPU(size_t N, return Status::OK(); } -gsl::span QLinearSoftmax::GetLookupTable(OpKernelContext* context, - gsl::span lookup_table_span, - size_t reduce_len) const { - gsl::span lookup_table = fixed_lookup_table_; +gsl::span QLinearSoftmax::GetLookupTable( + OpKernelContext* context, + gsl::span lookup_table_span, + size_t reduce_len) const { + gsl::span lookup_table = fixed_lookup_table_; if (fixed_lookup_table_.size() == 0) { lookup_table = lookup_table_span; const float X_scale = *(context->Input(1)->Data()); @@ -260,11 +265,11 @@ gsl::span QLinearSoftmax::GetLookupTable(OpKernelContext* contex // opset-12 and below Status QLinearSoftmax::ComputeInternal(OpKernelContext* context, const Tensor& input, Tensor& output, - gsl::span lookup_table, int axis, + gsl::span lookup_table, int axis, concurrency::ThreadPool* thread_pool) const { const auto* Y_scale_tensor = context->Input(3); const auto* Y_zp_tensor = context->Input(4); - const auto Y_scale = gsl::narrow_cast(1.0F / (*(Y_scale_tensor->Data()))); + const QLinearSoftmax::EXP_OUT_DTYPE Y_scale = std::floor(1.0F / (*(Y_scale_tensor->Data()))); const auto& X_shape = input.Shape(); const size_t N = X_shape.SizeToDimension(axis); const size_t D = X_shape.SizeFromDimension(axis); @@ -286,7 +291,7 @@ Status QLinearSoftmax::ComputeInternal(OpKernelContext* context, const Tensor& i // opset-13 and above Status QLinearSoftmax::ComputeImplOpset13(OpKernelContext* context, const Tensor& input, Tensor& output, - gsl::span lookup_table, + gsl::span lookup_table, concurrency::ThreadPool* thread_pool) const { const auto& X_shape = input.Shape(); size_t rank = X_shape.NumDimensions(); diff --git a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h index a90083cd40..08785f05a6 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h +++ b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h @@ -12,19 +12,25 @@ namespace contrib { class QLinearSoftmax final : public OpKernel { public: + using EXP_OUT_DTYPE = float; // or uint32_t if uint32_t is preferred. QLinearSoftmax(const OpKernelInfo& info); Status Compute(OpKernelContext* context) const override; private: - gsl::span GetLookupTable(OpKernelContext* context, gsl::span lookup_table_span, size_t reduce_len) const; + gsl::span GetLookupTable( + OpKernelContext* context, + gsl::span lookup_table_span, size_t reduce_len) const; - Status ComputeInternal(OpKernelContext* context, const Tensor& input, Tensor& output, gsl::span lookup_table, int axis, concurrency::ThreadPool* thread_pool) const; + Status ComputeInternal(OpKernelContext* context, const Tensor& input, + Tensor& output, gsl::span lookup_table, + int axis, concurrency::ThreadPool* thread_pool) const; Status ComputeImplOpset13(OpKernelContext* context, const Tensor& input, Tensor& output, - gsl::span lookup_table, concurrency::ThreadPool* thread_pool) const; + gsl::span lookup_table, + concurrency::ThreadPool* thread_pool) const; private: - std::vector fixed_lookup_table_; + std::vector fixed_lookup_table_; int axis_ = -1; int opset_ = 1; bool is_signed_{false}; diff --git a/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc b/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc index e0e7e4cc39..aaa3097932 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/generation_device_helper.cc @@ -468,8 +468,8 @@ Status GreedySearchProcessLogits( topk_indices)); #ifdef DEBUG_GENERATION - dumper->Print("topk_scores", *(topk_scores.get())); - dumper->Print("topk_indices", *(topk_indices.get())); + dumper->Print("topk_scores", topk_scores); + dumper->Print("topk_indices", topk_indices); #endif gsl::span next_token_indices = topk_indices.DataAsSpan(); diff --git a/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h b/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h index ce272b4886..1f1b63485c 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h +++ b/onnxruntime/contrib_ops/cpu/transformers/generation_shared.h @@ -51,7 +51,6 @@ struct IGreedySearchState { gsl::span sequence_lengths; // shape (batch_size) gsl::span next_positions; // shape (batch_size, num_beams). Next position value for position_ids. gsl::span eos_meet; // shape (batch_size) - gsl::span next_token_scores; // shape (batch_size, vocab_size) gsl::span next_tokens; // shape (batch_size) }; diff --git a/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc b/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc index 440d2eed75..088c59fac2 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/greedy_search.cc @@ -127,43 +127,43 @@ Status GreedySearch::Compute(OpKernelContext* ctx) const { // make a copy since we will update the parameters based on inputs later GreedySearchParameters parameters = parameters_; -if (parameters_.model_type == 0) { // GPT-2 + if (parameters_.model_type == 0) { // GPT-2 // Subgraph has constraint that the output is either float or float16 if (!gpt_subgraph_->IsOutputFloat16()) { GreedySearchGpt impl{ - *ctx_internal, - *decoder_session_state, - *gpt_subgraph_, - thread_pool, - cuda_stream_, - dumper_, - parameters, - GenerationCpuDeviceHelper::CreateGptInputs, - add_to_feeds_func_ ? add_to_feeds_func_ : GenerationCpuDeviceHelper::AddToFeeds, - topk_func_ ? topk_func_ : GenerationCpuDeviceHelper::TopK, - process_logits_func_ ? process_logits_func_ : GenerationCpuDeviceHelper::GreedySearchProcessLogits, - init_greedy_state_func_ ? init_greedy_state_func_ : GenerationCpuDeviceHelper::InitGreedyState, - device_copy_func_ ? device_copy_func_ : GenerationCpuDeviceHelper::DeviceCopy, - update_gpt_feeds_func_ ? update_gpt_feeds_func_ : GenerationCpuDeviceHelper::UpdateGptFeeds}; + *ctx_internal, + *decoder_session_state, + *gpt_subgraph_, + thread_pool, + cuda_stream_, + dumper_, + parameters, + GenerationCpuDeviceHelper::CreateGptInputs, + add_to_feeds_func_ ? add_to_feeds_func_ : GenerationCpuDeviceHelper::AddToFeeds, + topk_func_ ? topk_func_ : GenerationCpuDeviceHelper::TopK, + process_logits_func_ ? process_logits_func_ : GenerationCpuDeviceHelper::GreedySearchProcessLogits, + init_greedy_state_func_ ? init_greedy_state_func_ : GenerationCpuDeviceHelper::InitGreedyState, + device_copy_func_ ? device_copy_func_ : GenerationCpuDeviceHelper::DeviceCopy, + update_gpt_feeds_func_ ? update_gpt_feeds_func_ : GenerationCpuDeviceHelper::UpdateGptFeeds}; ORT_RETURN_IF_ERROR(impl.Initialize()); return impl.Execute(*decoder_feeds_fetches_manager_); } else { GreedySearchGpt impl{ - *ctx_internal, - *decoder_session_state, - *gpt_subgraph_, - thread_pool, - cuda_stream_, - dumper_, - parameters, - GenerationCpuDeviceHelper::CreateGptInputs, - add_to_feeds_func_ ? add_to_feeds_func_ : GenerationCpuDeviceHelper::AddToFeeds, - topk_func_ ? topk_func_ : GenerationCpuDeviceHelper::TopK, - process_logits_fp16_func_, - init_greedy_state_fp16_func_, - device_copy_func_, - update_gpt_feeds_fp16_func_}; + *ctx_internal, + *decoder_session_state, + *gpt_subgraph_, + thread_pool, + cuda_stream_, + dumper_, + parameters, + GenerationCpuDeviceHelper::CreateGptInputs, + add_to_feeds_func_ ? add_to_feeds_func_ : GenerationCpuDeviceHelper::AddToFeeds, + topk_func_ ? topk_func_ : GenerationCpuDeviceHelper::TopK, + process_logits_fp16_func_, + init_greedy_state_fp16_func_, + device_copy_func_, + update_gpt_feeds_fp16_func_}; ORT_RETURN_IF_ERROR(impl.Initialize()); return impl.Execute(*decoder_feeds_fetches_manager_); diff --git a/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h b/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h index 3c17c19043..a98df8c7e2 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h +++ b/onnxruntime/contrib_ops/cpu/transformers/greedy_search_impl_gpt.h @@ -16,19 +16,19 @@ template class GreedySearchGpt : public GreedySearchBase { public: GreedySearchGpt(OpKernelContextInternal& context, - const SessionState& decoder_session_state, - GptSubgraph& gpt_subgraph, - concurrency::ThreadPool* thread_pool, - void* cuda_stream, - IConsoleDumper* cuda_dumper, - GreedySearchParameters& params, - const GenerationDeviceHelper::CreateGptInputsFunc& create_inputs_func, - const GenerationDeviceHelper::AddToFeedsFunc& add_to_feeds_func, - const GenerationDeviceHelper::TopkFunc& topk_func, - const GenerationDeviceHelper::GreedySearchProcessLogitsFunc& process_logits_func, - const GenerationDeviceHelper::InitGreedyStateFunc& init_greedy_state_func, - const GenerationDeviceHelper::DeviceCopyFunc& device_copy_func, - const GenerationDeviceHelper::UpdateGptFeedsFunc& update_feeds_func) + const SessionState& decoder_session_state, + GptSubgraph& gpt_subgraph, + concurrency::ThreadPool* thread_pool, + void* cuda_stream, + IConsoleDumper* cuda_dumper, + GreedySearchParameters& params, + const GenerationDeviceHelper::CreateGptInputsFunc& create_inputs_func, + const GenerationDeviceHelper::AddToFeedsFunc& add_to_feeds_func, + const GenerationDeviceHelper::TopkFunc& topk_func, + const GenerationDeviceHelper::GreedySearchProcessLogitsFunc& process_logits_func, + const GenerationDeviceHelper::InitGreedyStateFunc& init_greedy_state_func, + const GenerationDeviceHelper::DeviceCopyFunc& device_copy_func, + const GenerationDeviceHelper::UpdateGptFeedsFunc& update_feeds_func) : GreedySearchBase(context, decoder_session_state, thread_pool, @@ -225,7 +225,7 @@ Status GreedySearchGpt::Execute(const FeedsFetchesManager& feeds_fetches_mana // Copy the sequences to output gsl::span output = output_sequences->MutableDataAsSpan(); for (int batch_id = 0; batch_id < parameters->batch_size; ++batch_id) { - auto batch_output = output.subspan(batch_id * parameters->max_length, parameters->max_length); + auto batch_output = output.subspan(batch_id * parameters->max_length, parameters->max_length); gsl::span sequence_source = greedy_state.sequences.GetSequence(batch_id); gsl::copy(sequence_source, batch_output); } diff --git a/onnxruntime/contrib_ops/cpu/transformers/greedy_search_parameters.cc b/onnxruntime/contrib_ops/cpu/transformers/greedy_search_parameters.cc index 5496511de5..fe42335201 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/greedy_search_parameters.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/greedy_search_parameters.cc @@ -37,8 +37,7 @@ void GreedySearchParameters::ParseFromInputs(OpKernelContext* context) { num_beams = static_cast(1); auto* repetition_penalty_tensor = context->Input(3); - repetition_penalty = repetition_penalty_tensor ? - static_cast(*repetition_penalty_tensor->Data()) : 1.0f; + repetition_penalty = repetition_penalty_tensor ? static_cast(*repetition_penalty_tensor->Data()) : 1.0f; ORT_ENFORCE(repetition_penalty > 0.0f, "repetition_penalty shall be greater than 0, got ", repetition_penalty); } diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.h b/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.h index 18edfc3740..cdda69d04d 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.h +++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_decoder.h @@ -17,8 +17,8 @@ class T5DecoderSubgraph : public Subgraph { const std::string& attribute_name, const GraphViewer& subgraph_in) : Subgraph(node_in, attribute_name, subgraph_in), has_hidden_state_(false) { - first_present_output_index_ = 1; - } + first_present_output_index_ = 1; + } // Create inputs for first inference of decoder subgraph. Status CreateInitialFeeds( diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_encoder.h b/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_encoder.h index 0651c5d946..7d2e44f952 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_encoder.h +++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_t5_encoder.h @@ -16,8 +16,8 @@ class T5EncoderSubgraph : public Subgraph { const onnxruntime::Node& node_in, const std::string& attribute_name, const GraphViewer& subgraph_in) : Subgraph(node_in, attribute_name, subgraph_in) { - first_present_output_index_ = 2; - } + first_present_output_index_ = 2; + } // Create inputs for first inference of subgraph. Status CreateInitialFeeds( diff --git a/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu new file mode 100644 index 0000000000..429265a545 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu @@ -0,0 +1,229 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/cu_inc/common.cuh" +#include "contrib_ops/cuda/bert/add_bias_transpose.h" + +namespace onnxruntime { +namespace cuda { + +struct __align__(8) Half4 { + half2 x; + half2 y; +}; + +__device__ __forceinline__ Half4 operator+(const Half4& a, const Half4& b) { + Half4 r; + r.x = a.x + b.x; + r.y = a.y + b.y; + return r; +} + +__device__ __forceinline__ float2 operator+(const float2& a, const float2& b) { + return make_float2(a.x + b.x, a.y + b.y); +} + +__device__ __forceinline__ float4 operator+(const float4& a, const float4& b) { + return make_float4(a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w); +} + +} // namespace cuda +} // namespace onnxruntime + +using namespace onnxruntime::cuda; + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +template +__global__ void AddBiasTransposeQKV(const T* input, const T* biases, T* output) { + // Input: BxSxMxNxH (Format 1) + // Output: MxBxNxSxH + // B is batch_size, S is sequence_length, M is number of matrices, N is num_heads, H is head_size + + int n = threadIdx.y; + int s = blockIdx.x; + int b = blockIdx.y; + int m = blockIdx.z; // matrix id + + const int head_size = blockDim.x; + const int num_heads = blockDim.y; + + const int sequence_length = gridDim.x; + const int batch_size = gridDim.y; + const int M = gridDim.z; + const int H = head_size; + const int NH = num_heads * head_size; + const int NHS = NH * sequence_length; + + int in_offset = n * head_size + (m + s * M) * NH + b * NHS * M; + const int out_offset = s * head_size + n * sequence_length * H + b * NHS + m * NHS * batch_size; + + const int h = threadIdx.x; + if (h < head_size) { + output[out_offset + h] = input[in_offset + h] + biases[m * NH + n * H + h]; + } +} + +template +__global__ void AddBiasTransposeQKVLarge(const int head_size, const T* input, const T* biases, T* output) { + int n = threadIdx.y; + int s = blockIdx.x; + int b = blockIdx.y; + int m = blockIdx.z; + + const int stride = blockDim.x; + const int num_heads = blockDim.y; + + const int sequence_length = gridDim.x; + const int batch_size = gridDim.y; + const int M = gridDim.z; + const int H = head_size; + const int NH = num_heads * H; + const int NHS = NH * sequence_length; + int in_offset = n * H + (m + s * M) * NH + b * NHS * M; + const int out_offset = s * H + n * sequence_length * H + b * NHS + m * NHS * batch_size; + + int h = threadIdx.x; + while (h < H) { + output[out_offset + h] = input[in_offset + h] + biases[m * NH + n * H + h]; + h += stride; + } +} + +template +__global__ void AddBiasTranspose(const T* input, const T* biases, T* output) { + // Input: MxBxSxNxH (Format 0) + // Output: MxBxNxSxH + // B is batch_size, S is sequence_length, M is number of matrices, N is num_heads, H is head_size + + int n = threadIdx.y; + int s = blockIdx.x; + int b = blockIdx.y; + int m = blockIdx.z; + + const int head_size = blockDim.x; + const int num_heads = blockDim.y; + const int sequence_length = gridDim.x; + const int batch_size = gridDim.y; + + const int H = head_size; + const int NH = num_heads * head_size; + const int NHS = NH * sequence_length; + + int in_offset = n * H + s * NH + (b + m * batch_size) * NHS; + const int out_offset = s * H + n * sequence_length * H + (b + m * batch_size) * NHS; + + const int h = threadIdx.x; + if (h < head_size) { + output[out_offset + h] = input[in_offset + h] + biases[m * NH + n * H + h]; + } +} + +template +__global__ void AddBiasTransposeLarge(const int head_size, const T* input, const T* biases, T* output) { + int n = threadIdx.y; + int s = blockIdx.x; + int b = blockIdx.y; + int m = blockIdx.z; + + const int stride = blockDim.x; + const int num_heads = blockDim.y; + + const int sequence_length = gridDim.x; + const int batch_size = gridDim.y; + + const int H = head_size; + const int NH = num_heads * H; + const int NHS = NH * sequence_length; + + int in_offset = n * H + s * NH + (b + m * batch_size) * NHS; + const int out_offset = (s + n * sequence_length) * H + (b + m * batch_size) * NHS; + + int h = threadIdx.x; + while (h < H) { + output[out_offset + h] = input[in_offset + h] + biases[m * NH + n * H + h]; + h += stride; + } +} + +template +void InvokeAddBiasTranspose( + cudaStream_t stream, const int num_matrices, const int format, const int max_threads_per_block, + const int batch_size, const int sequence_length, const int num_heads, const int head_size, + const T* input, const T* biases, T* output) { + const dim3 grid(sequence_length, batch_size, num_matrices); + if (head_size * num_heads <= max_threads_per_block) { + const dim3 block(head_size, num_heads, 1); + if (format == 1) { + AddBiasTransposeQKV<<>>(input, biases, output); + } else { + AddBiasTranspose<<>>(input, biases, output); + } + } else { + const dim3 block(CeilDiv(max_threads_per_block, num_heads), num_heads, 1); + if (format == 1) { + AddBiasTransposeQKVLarge<<>>(head_size, input, biases, output); + } else { + AddBiasTransposeLarge<<>>(head_size, input, biases, output); + } + } +} + +template <> +void LaunchAddBiasTranspose( + cudaStream_t stream, const int num_matrices, const int format, const int max_threads_per_block, + const int batch_size, const int sequence_length, const int num_heads, const int head_size, + const half* input, const half* biases, half* output, + bool enable_half4) { + if (enable_half4 && 0 == (head_size % 4)) { + const int H = head_size / 4; + const Half4* input2 = reinterpret_cast(input); + const Half4* biases2 = reinterpret_cast(biases); + Half4* output2 = reinterpret_cast(output); + InvokeAddBiasTranspose(stream, num_matrices, format, max_threads_per_block, + batch_size, sequence_length, num_heads, H, input2, biases2, output2); + } else if (0 == (head_size & 1)) { + const int H = head_size / 2; + const half2* input2 = reinterpret_cast(input); + const half2* biases2 = reinterpret_cast(biases); + half2* output2 = reinterpret_cast(output); + InvokeAddBiasTranspose(stream, num_matrices, format, max_threads_per_block, + batch_size, sequence_length, num_heads, H, input2, biases2, output2); + } else { + InvokeAddBiasTranspose(stream, num_matrices, format, max_threads_per_block, + batch_size, sequence_length, num_heads, head_size, input, biases, output); + } +} + +template <> +void LaunchAddBiasTranspose( + cudaStream_t stream, const int num_matrices, const int format, const int max_threads_per_block, + const int batch_size, const int sequence_length, const int num_heads, const int head_size, + const float* input, const float* biases, float* output, + bool /*enable_half4*/) { + if (0 == (head_size % 4)) { + const int H = head_size / 4; + const float4* input2 = reinterpret_cast(input); + const float4* biases2 = reinterpret_cast(biases); + float4* output2 = reinterpret_cast(output); + InvokeAddBiasTranspose(stream, num_matrices, format, max_threads_per_block, + batch_size, sequence_length, num_heads, H, input2, biases2, output2); + } else if (0 == (head_size & 1)) { + const int H = head_size / 2; + const float2* input2 = reinterpret_cast(input); + const float2* biases2 = reinterpret_cast(biases); + float2* output2 = reinterpret_cast(output); + + InvokeAddBiasTranspose(stream, num_matrices, format, max_threads_per_block, + batch_size, sequence_length, num_heads, H, input2, biases2, output2); + } else { + InvokeAddBiasTranspose(stream, num_matrices, format, max_threads_per_block, + batch_size, sequence_length, num_heads, head_size, input, biases, output); + } +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.h b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.h new file mode 100644 index 0000000000..e173c8888e --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/cuda/shared_inc/cuda_utils.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +// Fused kernel of Add (bias) and Transpose. +// Shape of inputs and outputs: +// input : (batch_size, sequence_length, num_matrices, num_heads, head_size) -- format 1 +// (num_matrices, batch_size, sequence_length, num_heads, head_size) -- format 0 +// biases: (num_matrices, num_heads * head_size) +// output: (num_matrices, batch_size, num_heads, sequence_length, head_size) +template +void LaunchAddBiasTranspose( + cudaStream_t stream, const int num_matrices, const int format, const int max_threads_per_block, + const int batch_size, const int sequence_length, const int num_heads, const int head_size, + const T* input, const T* biases, T* output, bool enable_half4); + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/attention.cc b/onnxruntime/contrib_ops/cuda/bert/attention.cc index 580e6b22e3..ba347b2381 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/attention.cc @@ -41,7 +41,13 @@ Status Attention::ComputeInternal(OpKernelContext* context) const { const Tensor* extra_add_qk = context->Input(5); auto& device_prop = GetDeviceProp(); - ORT_RETURN_IF_ERROR(CheckInputs(input->Shape(), weights->Shape(), bias->Shape(), mask_index, past, extra_add_qk, device_prop.maxThreadsPerBlock)); + ORT_RETURN_IF_ERROR(CheckInputs(input->Shape(), + weights->Shape(), + bias->Shape(), + mask_index, + past, + extra_add_qk, + device_prop.maxThreadsPerBlock)); // input shape (batch_size, sequence_length, input_hidden_size) const auto& shape = input->Shape(); @@ -77,46 +83,41 @@ Status Attention::ComputeInternal(OpKernelContext* context) const { CudaT one = ToCudaType::FromFloat(1.0f); CudaT zero = ToCudaType::FromFloat(0.0f); - // Bias shape is (N), broadcast using B(N, M) = 1 * bias(N, 1) x ones(1, M) + 0 * B. - // TODO: use custom kernel of expand to improve the performance. - CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, 1, &one, - reinterpret_cast(bias->Data()), n, - GetConstOnes(m), 1, - &zero, reinterpret_cast(gemm_buffer.get()), n, device_prop)); - // Gemm, note that CUDA assumes col-major, so result(N, M) = 1 * weights x input + 1 x B. CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, reinterpret_cast(weights->Data()), n, reinterpret_cast(input->Data()), k, - &one, reinterpret_cast(gemm_buffer.get()), n, device_prop)); + &zero, reinterpret_cast(gemm_buffer.get()), n, device_prop)); - size_t workSpaceSize = GetAttentionWorkspaceSize(element_size, batch_size, num_heads_, head_size, sequence_length, past_sequence_length); - auto temp_buffer = GetScratchBuffer(workSpaceSize); - if (!LaunchAttentionKernel( + size_t workSpaceSize = GetAttentionWorkspaceSize(element_size, + batch_size, + num_heads_, + head_size, + sequence_length, + past_sequence_length); + + auto work_space = GetScratchBuffer(workSpaceSize); + ORT_RETURN_IF_ERROR(LaunchAttentionKernel( device_prop, Stream(), - reinterpret_cast(gemm_buffer.get()), - nullptr == mask_index ? nullptr : mask_index->Data(), - nullptr == mask_index ? gsl::span() : mask_index->Shape().GetDims(), - output->MutableData(), + cublas, + element_size, batch_size, sequence_length, num_heads_, head_size, - temp_buffer.get(), - cublas, - element_size, - is_unidirectional_, past_sequence_length, + is_unidirectional_, + reinterpret_cast(gemm_buffer.get()), + bias->Data(), + nullptr == mask_index ? nullptr : mask_index->Data(), + nullptr == mask_index ? gsl::span() : mask_index->Shape().GetDims(), nullptr == past ? nullptr : past->Data(), nullptr == extra_add_qk ? nullptr : extra_add_qk->Data(), - nullptr == present ? nullptr : present->MutableData())) { - // Get last error to reset it to cudaSuccess. - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } + work_space.get(), + output->MutableData(), + nullptr == present ? nullptr : present->MutableData())); return Status::OK(); } diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_concat.cu b/onnxruntime/contrib_ops/cuda/bert/attention_concat.cu index 3f9ed50e89..684118dd59 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_concat.cu +++ b/onnxruntime/contrib_ops/cuda/bert/attention_concat.cu @@ -92,8 +92,7 @@ __global__ void ConcatTensorToTensorLarge(const int tensor_add_sequence_length, } } - -bool LaunchConcatTensorToTensor(cudaStream_t stream, +Status LaunchConcatTensorToTensor(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -109,10 +108,17 @@ bool LaunchConcatTensorToTensor(cudaStream_t stream, const int H = head_size / 2; if (H * num_heads <= max_threads_per_block) { const dim3 block(H, num_heads, 1); - ConcatTensorToTensor<<>>(sequence_length, reinterpret_cast(tensor_in), reinterpret_cast(tensor_add), reinterpret_cast(tensor_out)); + ConcatTensorToTensor<<>>(sequence_length, + reinterpret_cast(tensor_in), + reinterpret_cast(tensor_add), + reinterpret_cast(tensor_out)); } else { const dim3 block(max_threads_per_block / num_heads, num_heads, 1); - ConcatTensorToTensorLarge<<>>(sequence_length, H, reinterpret_cast(tensor_in), reinterpret_cast(tensor_add), reinterpret_cast(tensor_out)); + ConcatTensorToTensorLarge<<>>(sequence_length, + H, + reinterpret_cast(tensor_in), + reinterpret_cast(tensor_add), + reinterpret_cast(tensor_out)); } } else { if (head_size * num_heads <= max_threads_per_block) { @@ -120,14 +126,17 @@ bool LaunchConcatTensorToTensor(cudaStream_t stream, ConcatTensorToTensor<<>>(sequence_length, tensor_in, tensor_add, tensor_out); } else { const dim3 block(max_threads_per_block / num_heads, num_heads, 1); - ConcatTensorToTensorLarge<<>>(sequence_length, head_size, tensor_in, tensor_add, tensor_out); + ConcatTensorToTensorLarge<<>>(sequence_length, + head_size, + tensor_in, + tensor_add, + tensor_out); } - } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } -bool LaunchConcatTensorToTensor(cudaStream_t stream, +Status LaunchConcatTensorToTensor(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -143,19 +152,33 @@ bool LaunchConcatTensorToTensor(cudaStream_t stream, const int H = head_size / 4; if (H * num_heads <= max_threads_per_block) { const dim3 block(H, num_heads, 1); - ConcatTensorToTensor<<>>(sequence_length, reinterpret_cast(tensor_in), reinterpret_cast(tensor_add), reinterpret_cast(tensor_out)); + ConcatTensorToTensor<<>>(sequence_length, + reinterpret_cast(tensor_in), + reinterpret_cast(tensor_add), + reinterpret_cast(tensor_out)); } else { const dim3 block(max_threads_per_block / num_heads, num_heads, 1); - ConcatTensorToTensorLarge<<>>(sequence_length, H, reinterpret_cast(tensor_in), reinterpret_cast(tensor_add), reinterpret_cast(tensor_out)); + ConcatTensorToTensorLarge<<>>(sequence_length, + H, + reinterpret_cast(tensor_in), + reinterpret_cast(tensor_add), + reinterpret_cast(tensor_out)); } } else if (0 == (head_size & 1)) { const int H = head_size / 2; if (H * num_heads <= max_threads_per_block) { const dim3 block(H, num_heads, 1); - ConcatTensorToTensor<<>>(sequence_length, reinterpret_cast(tensor_in), reinterpret_cast(tensor_add), reinterpret_cast(tensor_out)); + ConcatTensorToTensor<<>>(sequence_length, + reinterpret_cast(tensor_in), + reinterpret_cast(tensor_add), + reinterpret_cast(tensor_out)); } else { const dim3 block(max_threads_per_block / num_heads, num_heads, 1); - ConcatTensorToTensorLarge<<>>(sequence_length, H, reinterpret_cast(tensor_in), reinterpret_cast(tensor_add), reinterpret_cast(tensor_out)); + ConcatTensorToTensorLarge<<>>(sequence_length, + H, + reinterpret_cast(tensor_in), + reinterpret_cast(tensor_add), + reinterpret_cast(tensor_out)); } } else { // this should be an "odd" case. probably not worth catching it in the half2 kernel. if (head_size * num_heads <= max_threads_per_block) { @@ -163,13 +186,17 @@ bool LaunchConcatTensorToTensor(cudaStream_t stream, ConcatTensorToTensor<<>>(sequence_length, tensor_in, tensor_add, tensor_out); } else { const dim3 block(max_threads_per_block / num_heads, num_heads, 1); - ConcatTensorToTensorLarge<<>>(sequence_length, head_size, tensor_in, tensor_add, tensor_out); + ConcatTensorToTensorLarge<<>>(sequence_length, + head_size, + tensor_in, + tensor_add, + tensor_out); } } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } -bool LaunchConcatPastToPresent(cudaStream_t stream, +Status LaunchConcatPastToPresent(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -180,20 +207,20 @@ bool LaunchConcatPastToPresent(cudaStream_t stream, const float* k_v, float* present) { return LaunchConcatTensorToTensor( - stream, - all_sequence_length, - sequence_length, - batch_size, - head_size, - num_heads, - max_threads_per_block, - 2, - past, - k_v, - present); + stream, + all_sequence_length, + sequence_length, + batch_size, + head_size, + num_heads, + max_threads_per_block, + 2, + past, + k_v, + present); } -bool LaunchConcatPastToPresent(cudaStream_t stream, +Status LaunchConcatPastToPresent(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -204,17 +231,17 @@ bool LaunchConcatPastToPresent(cudaStream_t stream, const half* k_v, half* present) { return LaunchConcatTensorToTensor( - stream, - all_sequence_length, - sequence_length, - batch_size, - head_size, - num_heads, - max_threads_per_block, - 2, - past, - k_v, - present); + stream, + all_sequence_length, + sequence_length, + batch_size, + head_size, + num_heads, + max_threads_per_block, + 2, + past, + k_v, + present); } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu index 835213e9b7..3f05b05daf 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu @@ -28,14 +28,12 @@ limitations under the License. #include "contrib_ops/cuda/bert/attention_impl.h" #include "contrib_ops/cuda/bert/attention_softmax.h" #include "contrib_ops/cuda/bert/transformer_common.h" +#include "contrib_ops/cuda/bert/add_bias_transpose.h" using namespace onnxruntime::cuda; using namespace cub; -#define CHECK_CUDA(expr) \ - if (!CUDA_CALL(expr)) { \ - return false; \ - } +#define CHECK_CUDA(expr) CUDA_RETURN_IF_ERROR(expr) namespace onnxruntime { namespace contrib { @@ -45,9 +43,13 @@ static size_t AlignTo(size_t a, size_t b) { return CeilDiv(a, b) * b; } -size_t GetAttentionScratchSize(size_t element_size, int batch_size, int num_heads, int sequence_length, int all_sequence_length) { - const size_t len = batch_size * num_heads * sequence_length * all_sequence_length; - const size_t bytes = len * element_size; +size_t GetAttentionScratchSize( + size_t element_size, + size_t batch_size, + size_t num_heads, + size_t sequence_length, + size_t all_sequence_length) { + const size_t bytes = element_size * batch_size * num_heads * sequence_length * all_sequence_length; const size_t alignment = 256; const size_t bytesAligned = AlignTo(bytes, alignment); @@ -56,24 +58,41 @@ size_t GetAttentionScratchSize(size_t element_size, int batch_size, int num_head size_t GetAttentionWorkspaceSize( size_t element_size, - int batch_size, - int num_heads, - int head_size, - int sequence_length, - int past_sequence_length) { - size_t qkv_size = 3 * batch_size * sequence_length * num_heads * head_size * element_size; - return qkv_size + 2 * GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, past_sequence_length + sequence_length); + size_t batch_size, + size_t num_heads, + size_t head_size, + size_t sequence_length, + size_t past_sequence_length) { + size_t qkv_size = element_size * 3 * batch_size * sequence_length * num_heads * head_size; + return qkv_size + 2 * GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, + past_sequence_length + sequence_length); } template -bool QkvToContext( - const cudaDeviceProp& prop, cublasHandle_t& cublas, cudaStream_t stream, - const int batch_size, const int sequence_length, const int num_heads, const int head_size, const size_t element_size, - const T* input, T* output, T* workspace, - const int* mask_index, gsl::span mask_index_dims, - bool is_unidirectional, int past_sequence_length, const T* past, const T* extra_add_qk, T* present, bool use_persistent_softmax) { +Status QkvToContext( + const cudaDeviceProp& prop, + cublasHandle_t& cublas, + cudaStream_t stream, + const int batch_size, + const int sequence_length, + const int num_heads, + const int head_size, + const size_t element_size, + const T* input, + const T* bias, + T* output, + T* workspace, + const int* mask_index, + gsl::span mask_index_dims, + bool is_unidirectional, + int past_sequence_length, + const T* past, + const T* extra_add_qk, + T* present, + bool use_persistent_softmax) { const int all_sequence_length = past_sequence_length + sequence_length; - const size_t bytes = GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, all_sequence_length); + const size_t bytes = GetAttentionScratchSize(element_size, batch_size, num_heads, + sequence_length, all_sequence_length); T* scratch1 = workspace; T* scratch2 = scratch1 + (bytes / element_size); T* scratch3 = scratch2 + (bytes / element_size); @@ -81,8 +100,17 @@ bool QkvToContext( const int max_threads_per_block = prop.maxThreadsPerBlock; // input should be BxSx3xNxH => scratch3: 3xBxNxSxH - if (!LaunchTransQkv(stream, 3, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, false, input, scratch3)) { - return false; + if (bias == nullptr) { + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 3, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, false, input, scratch3)); + } else { + const int format = 1; // BxSxMxNxH + const bool enable_half4 = true; + LaunchAddBiasTranspose(stream, 3, format, max_threads_per_block, batch_size, + sequence_length, num_heads, head_size, + input, bias, scratch3, + enable_half4); + CUDA_RETURN_IF_ERROR(cudaGetLastError()); } // now scratch3 has Q, K, V: each has size BxNxSxH @@ -101,9 +129,9 @@ bool QkvToContext( // past_v (BxNxS'xH) + v (BxNxSxH) => present_v (BxNxS*xH) const int present_size_per_batch = all_sequence_length * head_size; if (nullptr != present) { - if (!LaunchConcatPastToPresent(stream, all_sequence_length, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, past, k, present)) { - return false; - } + ORT_RETURN_IF_ERROR( + LaunchConcatPastToPresent(stream, all_sequence_length, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, past, k, present)); // update pointers to present_k and present_v. k = present; @@ -120,118 +148,132 @@ bool QkvToContext( float one = 1.0f; float zero = 0.f; - // For raw attention mask, the scalar if 1/sqrt(H) is moved to softmax computation. + // For raw attention mask, the scalar 1/sqrt(H) is moved to combine with softmax computation. float alpha = use_raw_attention_mask ? one : rsqrt_head_size; - if (!CUBLAS_CALL(cublasGemmStridedBatchedHelper( - cublas, CUBLAS_OP_T, CUBLAS_OP_N, all_sequence_length, sequence_length, head_size, &alpha, k, head_size, present_size_per_batch, - q, head_size, size_per_batch, &zero, scratch1, all_sequence_length, temp_matrix_size, batches, prop))) { - return false; - } + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper( + cublas, CUBLAS_OP_T, CUBLAS_OP_N, + all_sequence_length, sequence_length, head_size, + &alpha, k, head_size, present_size_per_batch, + q, head_size, size_per_batch, + &zero, scratch1, all_sequence_length, temp_matrix_size, batches, prop)); // apply softmax and store result P to scratch2: BxNxSxS* if (use_raw_attention_mask) { // 2d, 3d or 4d attention mask const int mask_dimension = static_cast(mask_index_dims.size()); - const int64_t max_sequence_length = mask_dimension == 4 ? mask_index_dims.at(3) : 0; + const int max_sequence_length = mask_dimension == 4 ? static_cast(mask_index_dims.at(3)) : 0; - T* persistent_softmax_workspace = scratch1; // replace Q*K' in place with masked score if persistent softmax is selected. - if (!ComputeSoftmaxWithRawMask(stream, all_sequence_length, sequence_length, batch_size, num_heads, mask_index, nullptr, extra_add_qk, scratch1, scratch2, - is_unidirectional, rsqrt_head_size, mask_dimension, static_cast(max_sequence_length), - use_persistent_softmax, persistent_softmax_workspace)) { - return false; - } + T* persistent_softmax_workspace = scratch1; // replace Q*K' in place with masked score for persistent softmax. + ORT_RETURN_IF_ERROR( + ComputeSoftmaxWithRawMask(stream, all_sequence_length, sequence_length, batch_size, num_heads, + mask_index, nullptr, extra_add_qk, scratch1, scratch2, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax, persistent_softmax_workspace)); } else if (nullptr != mask_index) { // 1d mask index ORT_ENFORCE(mask_index_dims.size() == 1); // mask_index has 1D shape: either (batch_size) or (2*batch_size). Only the later one has start postions. const int* mask_start = (mask_index_dims.at(0) > batch_size) ? mask_index + batch_size : nullptr; - if (!ComputeSoftmaxWithMask1D(stream, all_sequence_length, sequence_length, batch_size, num_heads, mask_index, mask_start, extra_add_qk, scratch1, scratch2, is_unidirectional)) { - return false; - } + ORT_RETURN_IF_ERROR(ComputeSoftmaxWithMask1D(stream, all_sequence_length, sequence_length, batch_size, num_heads, + mask_index, mask_start, extra_add_qk, scratch1, scratch2, is_unidirectional)); } else { // no mask - if (!ComputeSoftmax(stream, all_sequence_length, sequence_length, batch_size, num_heads, extra_add_qk, scratch1, scratch2, is_unidirectional)) { - return false; - } + ORT_RETURN_IF_ERROR( + ComputeSoftmax(stream, all_sequence_length, sequence_length, batch_size, num_heads, extra_add_qk, + scratch1, scratch2, is_unidirectional)); } // compute P*V (as V*P), and store in scratch3: BxNxSxH - if (!CUBLAS_CALL(cublasGemmStridedBatchedHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, head_size, sequence_length, all_sequence_length, &one, v, head_size, present_size_per_batch, - scratch2, all_sequence_length, temp_matrix_size, &zero, scratch3, head_size, size_per_batch, batches, prop))) { - return false; - } + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, + head_size, sequence_length, all_sequence_length, + &one, v, head_size, present_size_per_batch, + scratch2, all_sequence_length, temp_matrix_size, + &zero, scratch3, head_size, size_per_batch, batches, prop)); // scratch3 is BxNxSxH, transpose to output BxSxNxH - return LaunchTransCtx(stream, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, false, scratch3, output); + return LaunchTransCtx(stream, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, false, scratch3, output); } -bool LaunchAttentionKernel( +Status LaunchAttentionKernel( const cudaDeviceProp& prop, cudaStream_t stream, - const void* input, - const int* mask_index, - gsl::span mask_index_dims, - void* output, - const int batch_size, - const int sequence_length, - const int num_heads, - const int head_size, - void* workspace, cublasHandle_t& cublas, const size_t element_size, - bool is_unidirectional, + int batch_size, + int sequence_length, + int num_heads, + int head_size, int past_sequence_length, + bool is_unidirectional, + const void* input, + const void* bias, + const int* mask_index, + gsl::span mask_index_dims, const void* past, const void* extra_add_qk, + void* workspace, + void* output, void* present) { - - // For testing, environment variable ORT_TRANSFORMER_OPTIONS=1 could enable persistent softmax + // For testing, environment variable ORT_TRANSFORMER_OPTIONS=1 could enable persistent softmax used in Torch. const TransformerOptions* options = TransformerOptions::GetInstance(); bool use_persistent_softmax = options->IsPrecisionMode() && !options->DisablePersistentSoftmax(); if (element_size == 2) { - return QkvToContext(prop, cublas, stream, - batch_size, sequence_length, num_heads, head_size, element_size, - reinterpret_cast(input), reinterpret_cast(output), reinterpret_cast(workspace), - mask_index, mask_index_dims, is_unidirectional, - past_sequence_length, reinterpret_cast(past), reinterpret_cast(extra_add_qk), - reinterpret_cast(present), use_persistent_softmax); + return QkvToContext(prop, cublas, stream, batch_size, sequence_length, num_heads, head_size, element_size, + reinterpret_cast(input), + reinterpret_cast(bias), + reinterpret_cast(output), + reinterpret_cast(workspace), + mask_index, + mask_index_dims, + is_unidirectional, + past_sequence_length, + reinterpret_cast(past), + reinterpret_cast(extra_add_qk), + reinterpret_cast(present), + use_persistent_softmax); } else { - return QkvToContext(prop, cublas, stream, - batch_size, sequence_length, num_heads, head_size, element_size, - reinterpret_cast(input), reinterpret_cast(output), reinterpret_cast(workspace), - mask_index, mask_index_dims, is_unidirectional, - past_sequence_length, reinterpret_cast(past), reinterpret_cast(extra_add_qk), - reinterpret_cast(present), use_persistent_softmax); + return QkvToContext(prop, cublas, stream, batch_size, sequence_length, num_heads, head_size, element_size, + reinterpret_cast(input), + reinterpret_cast(bias), + reinterpret_cast(output), + reinterpret_cast(workspace), + mask_index, + mask_index_dims, + is_unidirectional, + past_sequence_length, + reinterpret_cast(past), + reinterpret_cast(extra_add_qk), + reinterpret_cast(present), + use_persistent_softmax); } } - template -bool DecoderQkvToContext( - const cudaDeviceProp& prop, - cudaStream_t stream, - cublasHandle_t& cublas, - const size_t element_size, - const int batch_size, - const int sequence_length, - const int kv_sequence_length, - const int num_heads, - const int head_size, - const bool static_kv, - const bool use_past, - const bool has_layer_state, - const bool has_key_padding_mask, - const T* gemm_query_buffer, - const T* gemm_kv_buffer, - const bool* key_padding_mask, - const T* key_cache, - const T* value_cache, - T* qkv_buffer, - T* workspace_buffer, - T* output, - T* new_key_cache, - T* new_value_cache) -{ +Status DecoderQkvToContext( + const cudaDeviceProp& prop, + cudaStream_t stream, + cublasHandle_t& cublas, + const size_t element_size, + const int batch_size, + const int sequence_length, + const int kv_sequence_length, + const int num_heads, + const int head_size, + const bool static_kv, + const bool use_past, + const bool has_layer_state, + const bool has_key_padding_mask, + const T* gemm_query_buffer, + const T* gemm_kv_buffer, + const bool* key_padding_mask, + const T* key_cache, + const T* value_cache, + T* qkv_buffer, + T* workspace_buffer, + T* output, + T* new_key_cache, + T* new_value_cache) { const int max_threads_per_block = prop.maxThreadsPerBlock; const int BN = batch_size * num_heads; const int BHN = BN * head_size; @@ -242,51 +284,59 @@ bool DecoderQkvToContext( T* temp_qkv_buffer = workspace_buffer; const T* q = qkv_buffer; - //transpose q and copy them to qkv_buffer - if (!LaunchTransQkv(stream, 1, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, true, gemm_query_buffer, qkv_buffer)) { - return false; - } + // transpose q and copy them to qkv_buffer + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_query_buffer, qkv_buffer)); const T* k = qkv_buffer + k_buffer_offset; const T* v = qkv_buffer + v_buffer_offset; if (!has_layer_state || !use_past) { if (!static_kv) { - //transpose kv and copy them to qkv_buffer - if (!LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)) { - return false; - } + // transpose kv and copy them to qkv_buffer + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)); } else { - //transpose kv and copy them to qkv_buffer - if (!LaunchTransQkv(stream, 2, kv_sequence_length, batch_size, head_size, num_heads, max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)) { - return false; - } + // transpose kv and copy them to qkv_buffer + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 2, kv_sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)); } } else { if (!static_kv) { - //transpose kv and copy them to temp_buffer - if (!LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, true, gemm_kv_buffer, temp_qkv_buffer)) { - return false; - } + // transpose kv and copy them to temp_buffer + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_kv_buffer, temp_qkv_buffer)); // concat cache-k with k and copy to qkv_buffer - if (nullptr != key_cache && !LaunchConcatTensorToTensor(stream, kv_sequence_length, sequence_length, batch_size, head_size, num_heads, - max_threads_per_block, 1, key_cache, temp_qkv_buffer, qkv_buffer + k_buffer_offset)) { - return false; + if (nullptr != key_cache) { + ORT_RETURN_IF_ERROR(LaunchConcatTensorToTensor(stream, kv_sequence_length, + sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, 1, + key_cache, + temp_qkv_buffer, + qkv_buffer + k_buffer_offset)); } // concat cache-v with v and copy to qkv_buffer - if (nullptr != value_cache && !LaunchConcatTensorToTensor(stream, kv_sequence_length, sequence_length, batch_size, head_size, num_heads, - max_threads_per_block, 1, value_cache, temp_qkv_buffer + k_buffer_offset, qkv_buffer + v_buffer_offset)) { - return false; + if (nullptr != value_cache) { + ORT_RETURN_IF_ERROR(LaunchConcatTensorToTensor(stream, kv_sequence_length, + sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, 1, + value_cache, + temp_qkv_buffer + k_buffer_offset, + qkv_buffer + v_buffer_offset)); } } } if (has_layer_state) { if (use_past && static_kv) { - CHECK_CUDA(cudaMemcpyAsync(new_key_cache, key_cache, kv_sequence_length * BHN * sizeof(T), cudaMemcpyDeviceToDevice, stream)); - CHECK_CUDA(cudaMemcpyAsync(new_value_cache, value_cache, kv_sequence_length * BHN * sizeof(T), cudaMemcpyDeviceToDevice, stream)); + CHECK_CUDA(cudaMemcpyAsync(new_key_cache, key_cache, kv_sequence_length * BHN * sizeof(T), + cudaMemcpyDeviceToDevice, stream)); + CHECK_CUDA(cudaMemcpyAsync(new_value_cache, value_cache, kv_sequence_length * BHN * sizeof(T), + cudaMemcpyDeviceToDevice, stream)); } else { - CHECK_CUDA(cudaMemcpyAsync(new_key_cache, k, kv_sequence_length * BHN * sizeof(T), cudaMemcpyDeviceToDevice, stream)); - CHECK_CUDA(cudaMemcpyAsync(new_value_cache, v, kv_sequence_length * BHN * sizeof(T), cudaMemcpyDeviceToDevice, stream)); + CHECK_CUDA(cudaMemcpyAsync(new_key_cache, k, kv_sequence_length * BHN * sizeof(T), + cudaMemcpyDeviceToDevice, stream)); + CHECK_CUDA(cudaMemcpyAsync(new_value_cache, v, kv_sequence_length * BHN * sizeof(T), + cudaMemcpyDeviceToDevice, stream)); } } @@ -308,128 +358,131 @@ bool DecoderQkvToContext( const int strideA = kv_sequence_length * head_size; const int strideB = sequence_length * head_size; if (use_past && static_kv) { - if (!CUBLAS_CALL(cublasGemmStridedBatchedHelper( - cublas, CUBLAS_OP_T, CUBLAS_OP_N, kv_sequence_length, sequence_length, head_size, &alpha, key_cache, head_size, strideA, - q, head_size, strideB, &zero, scratch1, kv_sequence_length, temp_matrix_size, BN, prop))) { - return false; - } + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper( + cublas, CUBLAS_OP_T, CUBLAS_OP_N, + kv_sequence_length, sequence_length, head_size, + &alpha, key_cache, head_size, strideA, + q, head_size, strideB, + &zero, scratch1, kv_sequence_length, temp_matrix_size, BN, prop)); } else { - if (!CUBLAS_CALL(cublasGemmStridedBatchedHelper( - cublas, CUBLAS_OP_T, CUBLAS_OP_N, kv_sequence_length, sequence_length, head_size, &alpha, k, head_size, strideA, - q, head_size, strideB, &zero, scratch1, kv_sequence_length, temp_matrix_size, BN, prop))) { - return false; - } + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper( + cublas, CUBLAS_OP_T, CUBLAS_OP_N, + kv_sequence_length, sequence_length, head_size, + &alpha, k, head_size, strideA, + q, head_size, strideB, + &zero, scratch1, kv_sequence_length, temp_matrix_size, BN, prop)); } + constexpr bool is_unidirectional = false; + const T* add_before_softmax = nullptr; if (has_key_padding_mask) { - if (!ComputeSoftmaxWithRawMask(stream, kv_sequence_length, sequence_length, batch_size, num_heads, nullptr, key_padding_mask, nullptr, scratch1, scratch2, - false, 1, 2, static_cast(0), false, nullptr)) { - return false; - } + constexpr int mask_dimension = 2; + constexpr int max_sequence_length = 0; + ORT_RETURN_IF_ERROR(ComputeSoftmaxWithRawMask(stream, kv_sequence_length, sequence_length, batch_size, num_heads, + nullptr, key_padding_mask, add_before_softmax, scratch1, scratch2, + is_unidirectional, 1.0f, mask_dimension, max_sequence_length, + false, nullptr)); } else { - if (!ComputeSoftmax(stream, kv_sequence_length, sequence_length, batch_size, num_heads, nullptr, scratch1, scratch2, false)) { - return false; - } + ORT_RETURN_IF_ERROR(ComputeSoftmax(stream, kv_sequence_length, sequence_length, batch_size, num_heads, + add_before_softmax, scratch1, scratch2, is_unidirectional)); } // compute P*V (as V*P), and store in scratch3: BxNxSxH if (use_past && static_kv) { - if (!CUBLAS_CALL(cublasGemmStridedBatchedHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, head_size, sequence_length, kv_sequence_length, &one, value_cache, head_size, strideA, - scratch2, kv_sequence_length, temp_matrix_size, &zero, scratch3, head_size, strideB, BN, prop))) { - return false; - } + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, + head_size, sequence_length, kv_sequence_length, + &one, value_cache, head_size, strideA, + scratch2, kv_sequence_length, temp_matrix_size, + &zero, scratch3, head_size, strideB, BN, prop)); } else { - if (!CUBLAS_CALL(cublasGemmStridedBatchedHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, head_size, sequence_length, kv_sequence_length, &one, v, head_size, strideA, - scratch2, kv_sequence_length, temp_matrix_size, &zero, scratch3, head_size, strideB, BN, prop))) { - return false; - } + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, + head_size, sequence_length, kv_sequence_length, + &one, v, head_size, strideA, + scratch2, kv_sequence_length, temp_matrix_size, + &zero, scratch3, head_size, strideB, BN, prop)); } // scratch3 is BxNxSxH, transpose to output SxBxNxH - return LaunchTransCtx(stream, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, true, scratch3, output); + return LaunchTransCtx(stream, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, scratch3, output); } - -bool LaunchDecoderAttentionKernel( - const cudaDeviceProp& prop, - cudaStream_t stream, - cublasHandle_t& cublas, - const size_t element_size, - const int batch_size, - const int sequence_length, - const int kv_sequence_length, - const int num_heads, - const int head_size, - const bool static_kv, - const bool use_past, - const bool has_layer_state, - const bool has_key_padding_mask, - const void* gemm_query_buffer, - const void* gemm_kv_buffer, - const bool* key_padding_mask, - const void* key_cache, - const void* value_cache, - void* qkv_buffer, - void* workspace_buffer, - void* output, - void* new_key_cache, - void* new_value_cache) -{ - +Status LaunchDecoderAttentionKernel( + const cudaDeviceProp& prop, + cudaStream_t stream, + cublasHandle_t& cublas, + const size_t element_size, + const int batch_size, + const int sequence_length, + const int kv_sequence_length, + const int num_heads, + const int head_size, + const bool static_kv, + const bool use_past, + const bool has_layer_state, + const bool has_key_padding_mask, + const void* gemm_query_buffer, + const void* gemm_kv_buffer, + const bool* key_padding_mask, + const void* key_cache, + const void* value_cache, + void* qkv_buffer, + void* workspace_buffer, + void* output, + void* new_key_cache, + void* new_value_cache) { if (element_size == 2) { return DecoderQkvToContext( - prop, - stream, - cublas, - element_size, - batch_size, - sequence_length, - kv_sequence_length, - num_heads, - head_size, - static_kv, - use_past, - has_layer_state, - has_key_padding_mask, - reinterpret_cast(gemm_query_buffer), - reinterpret_cast(gemm_kv_buffer), - key_padding_mask, - reinterpret_cast(key_cache), - reinterpret_cast(value_cache), - reinterpret_cast(qkv_buffer), - reinterpret_cast(workspace_buffer), - reinterpret_cast(output), - reinterpret_cast(new_key_cache), - reinterpret_cast(new_value_cache) - ); + prop, + stream, + cublas, + element_size, + batch_size, + sequence_length, + kv_sequence_length, + num_heads, + head_size, + static_kv, + use_past, + has_layer_state, + has_key_padding_mask, + reinterpret_cast(gemm_query_buffer), + reinterpret_cast(gemm_kv_buffer), + key_padding_mask, + reinterpret_cast(key_cache), + reinterpret_cast(value_cache), + reinterpret_cast(qkv_buffer), + reinterpret_cast(workspace_buffer), + reinterpret_cast(output), + reinterpret_cast(new_key_cache), + reinterpret_cast(new_value_cache)); } else { return DecoderQkvToContext( - prop, - stream, - cublas, - element_size, - batch_size, - sequence_length, - kv_sequence_length, - num_heads, - head_size, - static_kv, - use_past, - has_layer_state, - has_key_padding_mask, - reinterpret_cast(gemm_query_buffer), - reinterpret_cast(gemm_kv_buffer), - key_padding_mask, - reinterpret_cast(key_cache), - reinterpret_cast(value_cache), - reinterpret_cast(qkv_buffer), - reinterpret_cast(workspace_buffer), - reinterpret_cast(output), - reinterpret_cast(new_key_cache), - reinterpret_cast(new_value_cache) - ); + prop, + stream, + cublas, + element_size, + batch_size, + sequence_length, + kv_sequence_length, + num_heads, + head_size, + static_kv, + use_past, + has_layer_state, + has_key_padding_mask, + reinterpret_cast(gemm_query_buffer), + reinterpret_cast(gemm_kv_buffer), + key_padding_mask, + reinterpret_cast(key_cache), + reinterpret_cast(value_cache), + reinterpret_cast(qkv_buffer), + reinterpret_cast(workspace_buffer), + reinterpret_cast(output), + reinterpret_cast(new_key_cache), + reinterpret_cast(new_value_cache)); } } diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_impl.h b/onnxruntime/contrib_ops/cuda/bert/attention_impl.h index 9c1e6da801..6c83ff10b2 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_impl.h +++ b/onnxruntime/contrib_ops/cuda/bert/attention_impl.h @@ -9,80 +9,87 @@ namespace onnxruntime { namespace contrib { namespace cuda { -size_t GetAttentionScratchSize(size_t element_size, int batch_size, int num_heads, int sequence_length, int all_sequence_length); + +size_t GetAttentionScratchSize( + size_t element_size, + size_t batch_size, + size_t num_heads, + size_t sequence_length, + size_t all_sequence_length); size_t GetAttentionWorkspaceSize( size_t element_size, - int batchsize, - int num_heads, - int head_size, - int sequence_length, - int past_sequence_length); + size_t batchsize, + size_t num_heads, + size_t head_size, + size_t sequence_length, + size_t past_sequence_length); -bool LaunchAttentionKernel( - const cudaDeviceProp& prop, // Device Properties - cudaStream_t stream, // cuda stream - const void* input, // Input tensor - const int* mask_index, // Attention mask raw data or index (end position of each sequence, or end positions and start positions). NULL means no mask. - gsl::span mask_index_dims, // Mask index shape - void* output, // Output tensor - int batch_size, // Batch size (B) - int sequence_length, // Sequence length (S) - int num_heads, // Number of attention heads (N) - int head_size, // Hidden layer size per head (H) - void* workspace, // Temporary buffer - cublasHandle_t& cublas, // Cublas handle - const size_t element_size, // Element size of input tensor - bool is_unidirectional, // Whether there is unidirecitonal mask. - int past_sequence_length, // Sequence length in past state - const void* past, // Past state input - const void* extra_add_qk, // Additional Add - void* present // Present state output +Status LaunchAttentionKernel( + const cudaDeviceProp& prop, // Device Properties + cudaStream_t stream, // cuda stream + cublasHandle_t& cublas, // Cublas handle + const size_t element_size, // Element size of input tensor + int batch_size, // Batch size (B) + int sequence_length, // Sequence length (S) + int num_heads, // Number of attention heads (N) + int head_size, // Hidden layer size per head (H) + int past_sequence_length, // Sequence length in past state + bool is_unidirectional, // Whether there is unidirecitonal mask. + const void* input, // Input tensor + const void* bias, // Bias tensor + const int* mask_index, // Attention mask raw data or index. NULL means no mask. + gsl::span mask_index_dims, // Mask index shape + const void* past, // Past state input + const void* extra_add_qk, // Additional Add + void* workspace, // Temporary buffer + void* output, // Output tensor + void* present // Present state output ); -bool LaunchDecoderAttentionKernel( - const cudaDeviceProp& prop, // Device Properties - cudaStream_t stream, // Cuda stream - cublasHandle_t& cublas, // Cublas handle - const size_t element_size, // Element size of input tensor - const int batch_size, // Batch size (B) - const int sequence_length, // Sequence length (S) - const int kv_sequence_length, // Key/Value/Cache sequence length - const int num_heads, // Number of attention heads (N) - const int head_size, // Hidden layer size per head (H) - const bool static_kv, // Whether cross attention or not - const bool use_past, // Whether use cache or not - const bool has_layer_state, // Whether output cache or not - const bool has_key_padding_mask, // Whether use key_padding_mask or not - const void* gemm_query_buffer, // Query buffer - const void* gemm_kv_buffer, // Key and value buffer - const bool* key_padding_mask, // Key padding mask - const void* key_cache, // Input key cache - const void* value_cache, // Input value cache - void* qkv_buffer, // Temporary buffer - void* workspace_buffer, // Temporary buffer - void* output, // Output tensor - void* new_key_cache, // New_key_cache tensor - void* new_value_cache // New_value_cache tensor +Status LaunchDecoderAttentionKernel( + const cudaDeviceProp& prop, // Device Properties + cudaStream_t stream, // Cuda stream + cublasHandle_t& cublas, // Cublas handle + const size_t element_size, // Element size of input tensor + const int batch_size, // Batch size (B) + const int sequence_length, // Sequence length (S) + const int kv_sequence_length, // Key/Value/Cache sequence length + const int num_heads, // Number of attention heads (N) + const int head_size, // Hidden layer size per head (H) + const bool static_kv, // Whether cross attention or not + const bool use_past, // Whether use cache or not + const bool has_layer_state, // Whether output cache or not + const bool has_key_padding_mask, // Whether use key_padding_mask or not + const void* gemm_query_buffer, // Query buffer + const void* gemm_kv_buffer, // Key and value buffer + const bool* key_padding_mask, // Key padding mask + const void* key_cache, // Input key cache + const void* value_cache, // Input value cache + void* qkv_buffer, // Temporary buffer + void* workspace_buffer, // Temporary buffer + void* output, // Output tensor + void* new_key_cache, // New_key_cache tensor + void* new_value_cache // New_value_cache tensor ); -bool LaunchTransCtx(cudaStream_t stream, +Status LaunchTransCtx(cudaStream_t stream, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const float* input, float* output); -bool LaunchTransCtx(cudaStream_t stream, +Status LaunchTransCtx(cudaStream_t stream, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const half* input, half* output); -bool LaunchTransQkv(cudaStream_t stream, const int matrix_num, +Status LaunchTransQkv(cudaStream_t stream, const int matrix_num, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const float* input, float* output); -bool LaunchTransQkv(cudaStream_t stream, const int matrix_num, +Status LaunchTransQkv(cudaStream_t stream, const int matrix_num, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const half* input, half* output); -bool LaunchConcatTensorToTensor(cudaStream_t stream, +Status LaunchConcatTensorToTensor(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -94,7 +101,7 @@ bool LaunchConcatTensorToTensor(cudaStream_t stream, const float* tensor_add, float* tensor_out); -bool LaunchConcatTensorToTensor(cudaStream_t stream, +Status LaunchConcatTensorToTensor(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -106,7 +113,7 @@ bool LaunchConcatTensorToTensor(cudaStream_t stream, const half* tensor_add, half* tensor_out); -bool LaunchConcatPastToPresent(cudaStream_t stream, +Status LaunchConcatPastToPresent(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, @@ -117,7 +124,7 @@ bool LaunchConcatPastToPresent(cudaStream_t stream, const float* k_v, float* present); -bool LaunchConcatPastToPresent(cudaStream_t stream, +Status LaunchConcatPastToPresent(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_softmax.h b/onnxruntime/contrib_ops/cuda/bert/attention_softmax.h index d5f908b922..1ad908a31f 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_softmax.h +++ b/onnxruntime/contrib_ops/cuda/bert/attention_softmax.h @@ -50,6 +50,8 @@ __device__ inline void Softmax(const int all_sequence_length, float thread_data_max(-CUDART_INF_F); + const bool no_add = (add_before_softmax == nullptr); + // e^x is represented as infinity if x is large enough, like 100.f. // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if one or more item are large enough. // a math transform as below is leveraged to get a stable softmax: @@ -58,7 +60,7 @@ __device__ inline void Softmax(const int all_sequence_length, for (int i = threadIdx.x; i < valid_end; i += TPB) { if (i >= valid_start) { const int index = offset + i; - float input_at_idx = add_before_softmax == nullptr ? float(input[index]) : float(input[index] + add_before_softmax[index]); + float input_at_idx = no_add ? float(input[index]) : float(input[index] + add_before_softmax[index]); if (thread_data_max < input_at_idx) { thread_data_max = input_at_idx; } @@ -77,7 +79,7 @@ __device__ inline void Softmax(const int all_sequence_length, for (int i = threadIdx.x; i < valid_end; i += TPB) { if (i >= valid_start) { const int index = offset + i; - float val = add_before_softmax == nullptr ? input[index] : input[index] + add_before_softmax[index]; + float val = no_add ? input[index] : input[index] + add_before_softmax[index]; thread_data_sum += expf(val - max_block); } } @@ -90,7 +92,7 @@ __device__ inline void Softmax(const int all_sequence_length, for (int i = threadIdx.x; i < all_sequence_length; i += TPB) { const int index = offset + i; - float input_at_idx = add_before_softmax == nullptr ? float(input[index]) : float(input[index] + add_before_softmax[index]); + float input_at_idx = no_add ? float(input[index]) : float(input[index] + add_before_softmax[index]); const float val = (i >= valid_start && i < valid_end) ? expf(input_at_idx - max_block) * sum_reverse_block : 0.f; output[index] = T(val); } @@ -122,7 +124,8 @@ __device__ inline void SoftmaxSmall(const int all_sequence_length, if (is_unidirectional) { int end_unid = all_sequence_length - sequence_length + (blockIdx.x % sequence_length) + 1; if (end_unid <= valid_start) { - // In this situation, mask of [0, end_unid) and [valid_start, valid_end) has -10000, and [end_unid, valid_start) and [valid_end, all_seq_len) has -20000. + // In this situation, mask of [0, end_unid) and [valid_start, valid_end) has -10000, + // and [end_unid, valid_start) and [valid_end, all_seq_len) has -20000. // So [0, end_unid) will also have value after softmax. is_valid = threadIdx.x < end_unid; } else { @@ -136,7 +139,8 @@ __device__ inline void SoftmaxSmall(const int all_sequence_length, // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if one or more item are large enough. // a math transform as below is leveraged to get a stable softmax: // e^xi/(e^x1 + ...e^xn) = e^(xi - max) / (e^(x1 - max) + ... + e^(xn - max)) - float input_data = add_before_softmax == nullptr ? float(input[index]) : float(input[index] + add_before_softmax[index]); + const bool no_add = (add_before_softmax == nullptr); + float input_data = no_add ? float(input[index]) : float(input[index] + add_before_softmax[index]); float thread_data_max = is_valid ? input_data : float(-CUDART_INF_F); const auto max = BlockReduce(tmp_storage).Reduce(thread_data_max, cub::Max(), end); @@ -197,7 +201,7 @@ __device__ inline void SoftmaxWithRawMaskSmall(const int all_sequence_length, const int sequence_index = blockIdx.x % sequence_length; if (is_unidirectional) { - int from_index = all_sequence_length - sequence_length + sequence_index; // offset of from token in all sequence length. + int from_index = all_sequence_length - sequence_length + sequence_index; // offset in all sequence length. if (threadIdx.x > from_index) { thread_data = -10000.0f; } @@ -210,7 +214,8 @@ __device__ inline void SoftmaxWithRawMaskSmall(const int all_sequence_length, } else if (mask_dimension == 3) { mask_offset = (batch_index * sequence_length + sequence_index) * all_sequence_length + threadIdx.x; } else if (mask_dimension == 4) { - mask_offset = (batch_index * max_sequence_length + all_sequence_length - sequence_length + sequence_index) * max_sequence_length + threadIdx.x; + int from_index = all_sequence_length - sequence_length + sequence_index; + mask_offset = (batch_index * max_sequence_length + from_index) * max_sequence_length + threadIdx.x; } if (nullptr == key_padding_mask) { @@ -255,50 +260,75 @@ __device__ inline void SoftmaxWithRawMaskSmall(const int all_sequence_length, } template -__global__ void SoftmaxKernelSmall(const int all_sequence_length, const int sequence_length, const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { - SoftmaxSmall(all_sequence_length, sequence_length, all_sequence_length, 0, add_before_softmax, input, output, is_unidirectional); +__global__ void SoftmaxKernelSmall(const int all_sequence_length, + const int sequence_length, + const T* add_before_softmax, + const T* input, + T* output, + bool is_unidirectional) { + SoftmaxSmall(all_sequence_length, sequence_length, all_sequence_length, 0, + add_before_softmax, input, output, is_unidirectional); } template -__global__ void SoftmaxKernel(const int all_sequence_length, const int sequence_length, const T* add_before_softmax, const T* input, T* output) { - Softmax(all_sequence_length, sequence_length, all_sequence_length, 0, add_before_softmax, input, output); +__global__ void SoftmaxKernel(const int all_sequence_length, + const int sequence_length, + const T* add_before_softmax, + const T* input, + T* output) { + Softmax(all_sequence_length, sequence_length, all_sequence_length, 0, + add_before_softmax, input, output); } template -bool ComputeSoftmax( - cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, - const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { +Status ComputeSoftmax(cudaStream_t stream, const int all_sequence_length, const int sequence_length, + const int batch_size, const int num_heads, + const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { const dim3 grid(sequence_length * num_heads, batch_size, 1); if (all_sequence_length <= 32) { const int blockSize = 32; - SoftmaxKernelSmall<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + SoftmaxKernelSmall<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 64) { const int blockSize = 64; - SoftmaxKernelSmall<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + SoftmaxKernelSmall<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 128) { const int blockSize = 128; - SoftmaxKernelSmall<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + SoftmaxKernelSmall<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 256) { const int blockSize = 256; - SoftmaxKernelSmall<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + SoftmaxKernelSmall<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 512) { const int blockSize = 512; - SoftmaxKernelSmall<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + SoftmaxKernelSmall<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 1024) { const int blockSize = 1024; - SoftmaxKernelSmall<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + SoftmaxKernelSmall<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (!is_unidirectional) { const int blockSize = 1024; - SoftmaxKernel<<>>(all_sequence_length, sequence_length, add_before_softmax, input, output); + SoftmaxKernel<<>>( + all_sequence_length, sequence_length, add_before_softmax, input, output); } else { - ORT_THROW("Attention CUDA operator does not support total sequence length > 1024."); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Attention CUDA operator does not support total sequence length > 1024."); } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } template -__global__ void MaskedSoftmaxKernelSmall(const int all_sequence_length, const int sequence_length, const int* mask_end, const int* mask_start, const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { +__global__ void MaskedSoftmaxKernelSmall(const int all_sequence_length, + const int sequence_length, + const int* mask_end, + const int* mask_start, + const T* add_before_softmax, + const T* input, + T* output, + bool is_unidirectional) { __shared__ int start_position; __shared__ int end_position; @@ -315,11 +345,17 @@ __global__ void MaskedSoftmaxKernelSmall(const int all_sequence_length, const in } __syncthreads(); - SoftmaxSmall(all_sequence_length, sequence_length, end_position, start_position, add_before_softmax, input, output, is_unidirectional); + SoftmaxSmall(all_sequence_length, sequence_length, end_position, start_position, + add_before_softmax, input, output, is_unidirectional); } template -__global__ void MaskedSoftmaxKernel(const int all_sequence_length, const int sequence_length, const int* mask_end, const int* mask_start, const T* add_before_softmax, const T* input, T* output) { +__global__ void MaskedSoftmaxKernel(const int all_sequence_length, + const int sequence_length, + const int* mask_end, + const int* mask_start, + const T* add_before_softmax, + const T* input, T* output) { __shared__ int start_position; __shared__ int end_position; @@ -336,99 +372,164 @@ __global__ void MaskedSoftmaxKernel(const int all_sequence_length, const int seq } __syncthreads(); - Softmax(all_sequence_length, sequence_length, end_position, start_position, add_before_softmax, input, output); + Softmax(all_sequence_length, sequence_length, end_position, start_position, + add_before_softmax, input, output); } template -__global__ void SoftmaxWithRawMaskSmallKernel(const int all_sequence_length, const int sequence_length, const int* attention_mask, const bool* key_padding_mask, const T* add_before_softmax, const T* input, - T* output, const bool is_unidirectional, const float rsqrt_head_size, const int mask_dimension, const int max_sequence_length, +__global__ void SoftmaxWithRawMaskSmallKernel(const int all_sequence_length, + const int sequence_length, + const int* attention_mask, + const bool* key_padding_mask, + const T* add_before_softmax, + const T* input, + T* output, + const bool is_unidirectional, + const float rsqrt_head_size, + const int mask_dimension, + const int max_sequence_length, const bool skip_softmax) { - SoftmaxWithRawMaskSmall(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, output, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, skip_softmax); + SoftmaxWithRawMaskSmall( + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, output, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + skip_softmax); } template -bool ComputeSoftmaxWithMask1D(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, - const int* mask_index, const int* mask_start, const T* add_before_softmax, const T* input, T* output, const bool is_unidirectional) { +Status ComputeSoftmaxWithMask1D(cudaStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int num_heads, + const int* mask_index, + const int* mask_start, + const T* add_before_softmax, + const T* input, + T* output, + const bool is_unidirectional) { const dim3 grid(sequence_length * num_heads, batch_size, 1); if (all_sequence_length <= 32) { const int blockSize = 32; MaskedSoftmaxKernelSmall - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 64) { const int blockSize = 64; MaskedSoftmaxKernelSmall - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 128) { const int blockSize = 128; MaskedSoftmaxKernelSmall - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 256) { const int blockSize = 256; MaskedSoftmaxKernelSmall - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 512) { const int blockSize = 512; MaskedSoftmaxKernelSmall - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 1024) { const int blockSize = 1024; MaskedSoftmaxKernelSmall - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (!is_unidirectional) { const int blockSize = 1024; MaskedSoftmaxKernel - <<>>(all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output); + <<>>(all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output); } else { - ORT_THROW("Attention CUDA operator does not support total sequence length > 1024."); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Attention CUDA operator does not support total sequence length > 1024."); } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } template -bool ComputeSoftmaxWithRawMask(cudaStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, - const int* attention_mask, const bool* key_padding_mask, const T* add_before_softmax, const T* input, T* output, const bool is_unidirectional, - const float rsqrt_head_size, const int mask_dimension, const int max_sequence_length, const bool use_persistent_softmax, T* persistent_softmax_workspace) { +Status ComputeSoftmaxWithRawMask(cudaStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int num_heads, + const int* attention_mask, + const bool* key_padding_mask, + const T* add_before_softmax, + const T* input, + T* output, + const bool is_unidirectional, + const float rsqrt_head_size, + const int mask_dimension, + const int max_sequence_length, + const bool use_persistent_softmax, + T* persistent_softmax_workspace) { const dim3 grid(sequence_length * num_heads, batch_size, 1); T* out = use_persistent_softmax ? persistent_softmax_workspace : output; if (all_sequence_length <= 32) { const int blockSize = 32; SoftmaxWithRawMaskSmallKernel - <<>>(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + <<>>(all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 64) { const int blockSize = 64; SoftmaxWithRawMaskSmallKernel - <<>>(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + <<>>(all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 128) { const int blockSize = 128; SoftmaxWithRawMaskSmallKernel - <<>>(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + <<>>(all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 256) { const int blockSize = 256; SoftmaxWithRawMaskSmallKernel - <<>>(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + <<>>(all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 512) { const int blockSize = 512; SoftmaxWithRawMaskSmallKernel - <<>>(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + <<>>(all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 1024) { const int blockSize = 1024; SoftmaxWithRawMaskSmallKernel - <<>>(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + <<>>(all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else { - ORT_THROW("Attention CUDA operator does not support total sequence length > 1024."); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Attention CUDA operator does not support total sequence length > 1024."); } if (use_persistent_softmax) { - dispatch_warpwise_softmax_forward(stream, output, persistent_softmax_workspace, all_sequence_length, all_sequence_length, batch_size * num_heads * sequence_length); + dispatch_warpwise_softmax_forward(stream, + output, + persistent_softmax_workspace, + all_sequence_length, + all_sequence_length, + batch_size * num_heads * sequence_length); } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } - } // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_transpose.cu b/onnxruntime/contrib_ops/cuda/bert/attention_transpose.cu index ce203ac8d8..d34c49feec 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_transpose.cu +++ b/onnxruntime/contrib_ops/cuda/bert/attention_transpose.cu @@ -92,7 +92,7 @@ __global__ void TransposeCtxLarge(const int H, const bool reversed_bs, const T* } } -bool LaunchTransCtx(cudaStream_t stream, +Status LaunchTransCtx(cudaStream_t stream, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const float* input, float* output) { const dim3 grid(sequence_length, batch_size, 1); @@ -116,10 +116,10 @@ bool LaunchTransCtx(cudaStream_t stream, TransposeCtxLarge<<>>(head_size, reversed_bs, input, output); } } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } -bool LaunchTransCtx(cudaStream_t stream, +Status LaunchTransCtx(cudaStream_t stream, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const half* input, half* output) { const dim3 grid(sequence_length, batch_size, 1); @@ -155,7 +155,7 @@ bool LaunchTransCtx(cudaStream_t stream, } } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } template @@ -229,7 +229,7 @@ __global__ void TransposeQKVLarge(const int H, const bool reversed_bs, const T* } } -bool LaunchTransQkv(cudaStream_t stream, const int matrix_num, +Status LaunchTransQkv(cudaStream_t stream, const int matrix_num, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const float* input, float* output) { const dim3 grid(sequence_length, batch_size, matrix_num); @@ -252,12 +252,11 @@ bool LaunchTransQkv(cudaStream_t stream, const int matrix_num, const dim3 block(max_threads_per_block / num_heads, num_heads, 1); TransposeQKVLarge<<>>(head_size, reversed_bs, input, output); } - } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } -bool LaunchTransQkv(cudaStream_t stream, const int matrix_num, +Status LaunchTransQkv(cudaStream_t stream, const int matrix_num, const int sequence_length, const int batch_size, const int head_size, const int num_heads, const int max_threads_per_block, const bool reversed_bs, const half* input, half* output) { const dim3 grid(sequence_length, batch_size, matrix_num); @@ -292,10 +291,9 @@ bool LaunchTransQkv(cudaStream_t stream, const int matrix_num, TransposeQKVLarge<<>>(head_size, reversed_bs, input, output); } } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } - } // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc b/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc index 411f3ff6f2..0cd0be1714 100644 --- a/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc @@ -85,7 +85,7 @@ Status CheckInputs(const TensorShape& query_shape, return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "q_weights shall have shape (hidden size, hidden size)"); } - if (kv_weights_dims[0] != hidden_size || kv_weights_dims[1] != 2 * hidden_size) { + if (kv_weights_dims[0] != hidden_size || kv_weights_dims[1] != static_cast(2 * hidden_size)) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "kv_weights shall have shape (hidden size, 2 * hidden size)"); } @@ -104,7 +104,8 @@ Status CheckInputs(const TensorShape& query_shape, const auto& kp_mask_dims = key_padding_mask->Shape().GetDims(); if (kp_mask_dims.size() != 2) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'key_padding_mask' is expected to have 2 dimension, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Input 'key_padding_mask' is expected to have 2 dimension, got ", kp_mask_dims.size()); } @@ -123,7 +124,8 @@ Status CheckInputs(const TensorShape& query_shape, } if (kp_mask_dims[1] != key_length) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "key_padding_mask shall have same sequence length as generated key"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "key_padding_mask shall have same sequence length as generated key"); } } @@ -188,10 +190,14 @@ Status DecoderAttention::ComputeInternal(OpKernelContext* context) const { // Copy static_kv, use_past and has_layer_state to CPU auto pinned_buffer = AllocateBufferOnCPUPinned(4 * sizeof(bool)); bool* kernel_state_pinned = reinterpret_cast(pinned_buffer.get()); - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned, static_kv->Data(), sizeof(bool), cudaMemcpyDeviceToHost, stream)); - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned + 1, use_past->Data(), sizeof(bool), cudaMemcpyDeviceToHost, stream)); - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned + 2, has_layer_state->Data(), sizeof(bool), cudaMemcpyDeviceToHost, stream)); - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned + 3, has_key_padding_mask->Data(), sizeof(bool), cudaMemcpyDeviceToHost, stream)); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned, static_kv->Data(), sizeof(bool), + cudaMemcpyDeviceToHost, stream)); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned + 1, use_past->Data(), sizeof(bool), + cudaMemcpyDeviceToHost, stream)); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned + 2, has_layer_state->Data(), sizeof(bool), + cudaMemcpyDeviceToHost, stream)); + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(kernel_state_pinned + 3, has_key_padding_mask->Data(), sizeof(bool), + cudaMemcpyDeviceToHost, stream)); // Create an event to make sure the async copy is finished before reading the data. AutoDestoryCudaEvent new_event; @@ -253,11 +259,13 @@ Status DecoderAttention::ComputeInternal(OpKernelContext* context) const { has_key_padding_mask_) ); - // calcualte q + // calculate q gemm_query_buffer_p = GetScratchBuffer(batch_size * sequence_length * hidden_size * element_size); m = sequence_length * batch_size; n = hidden_size; k = hidden_size; + + // TODO(tianleiwu): fuse bias and transpose // broadcast bias for query: (h2, S*B) CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, 1, &one, @@ -342,15 +350,18 @@ Status DecoderAttention::ComputeInternal(OpKernelContext* context) const { } } - auto qkv_buffer_p = GetScratchBuffer(batch_size * (sequence_length + 2 * kv_sequence_length) * hidden_size * element_size); - auto workspace_p = GetScratchBuffer(2 * batch_size * sequence_length * num_heads_ * element_size * (2 * head_size + kv_sequence_length)); + size_t bytes = element_size * batch_size * (sequence_length + 2 * kv_sequence_length) * hidden_size; + auto qkv_buffer_p = GetScratchBuffer(bytes); + + bytes = element_size * 2 * batch_size * sequence_length * num_heads_ * (2 * head_size + kv_sequence_length); + auto workspace_p = GetScratchBuffer(bytes); Tensor* output(context->Output(0, query_shape)); TensorShape new_cache_shape({batch_size, num_heads_, kv_sequence_length, head_size}); Tensor* new_key_cache(context->Output(1, new_cache_shape)); Tensor* new_value_cache(context->Output(2, new_cache_shape)); - if (!LaunchDecoderAttentionKernel( + return LaunchDecoderAttentionKernel( device_prop, stream, cublas, @@ -373,13 +384,7 @@ Status DecoderAttention::ComputeInternal(OpKernelContext* context) const { workspace_p.get(), output->MutableData(), nullptr == new_key_cache ? nullptr : new_key_cache->MutableData(), - nullptr == new_value_cache ? nullptr : new_value_cache->MutableData())) { - // Get last error to reset it to cudaSuccess. - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + nullptr == new_value_cache ? nullptr : new_value_cache->MutableData()); } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc index 1dfca4f459..f1e1696acc 100644 --- a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc +++ b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm.cc @@ -43,8 +43,8 @@ Status EmbedLayerNorm::ComputeInternal(OpKernelContext* context) const { const Tensor* segment_embedding = context->Input(4); // optional. nullptr if it's distill-bert const Tensor* gamma = context->Input(5); const Tensor* beta = context->Input(6); - const Tensor* mask = context->Input(7); // optional. nullptr if not provided - const Tensor* position_ids = context->Input(8); // optional. nullptr if not provided + const Tensor* mask = context->Input(7); // optional. nullptr if not provided + const Tensor* position_ids = context->Input(8); // optional. nullptr if not provided const auto& input_dims = input_ids->Shape().GetDims(); int64_t hidden_size = word_embedding->Shape()[1]; @@ -61,7 +61,7 @@ Status EmbedLayerNorm::ComputeInternal(OpKernelContext* context) const { int sequence_length = static_cast(input_dims[1]); size_t element_size = sizeof(T); - if (!LaunchEmbedLayerNormKernel( + return LaunchEmbedLayerNormKernel( Stream(), output->MutableData(), mask_index->MutableData(), @@ -79,15 +79,9 @@ Status EmbedLayerNorm::ComputeInternal(OpKernelContext* context) const { sequence_length, element_size, embedding_sum == nullptr ? nullptr : embedding_sum->MutableData(), - position_ids == nullptr ? nullptr : position_ids->Data())) { - // Get last error to reset it to cudaSuccess. - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + position_ids == nullptr ? nullptr : position_ids->Data()); } -} //namespace cuda +} // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.cu b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.cu index f718bc643e..49437bd49e 100644 --- a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.cu @@ -85,7 +85,11 @@ __global__ void MaskIndexKernel(int sequence_length, const int* mask, int* mask_ } } -inline bool ComputeMaskIndex(cudaStream_t stream, const int sequence_length, const int batch_size, const int* mask, int* mask_index) { +inline Status ComputeMaskIndex(cudaStream_t stream, + const int sequence_length, + const int batch_size, + const int* mask, + int* mask_index) { // Mask idx is of length batch_size and assumes the valid region is contiguous starting // from the beginning of the sequence @@ -100,7 +104,7 @@ inline bool ComputeMaskIndex(cudaStream_t stream, const int sequence_length, con MaskIndexKernel<256><<>>(sequence_length, mask, mask_index); } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } template @@ -168,7 +172,7 @@ __global__ void EmbedLayerNormKernel( } template -bool EmbedSkipLayerNorm( +Status EmbedSkipLayerNorm( cudaStream_t stream, int hidden_size, int batch_size, int sequence_length, const int* input_ids, const int* segment_ids, const T* beta, const T* gamma, const T* word_embedding, const T* position_embedding, const T* segment_embedding, @@ -178,12 +182,14 @@ bool EmbedSkipLayerNorm( const dim3 block(tpb, 1, 1); EmbedLayerNormKernel - <<>>(hidden_size, input_ids, segment_ids, beta, gamma, word_embedding, position_embedding, segment_embedding, epsilon, output, embedding_sum, position_ids); + <<>>(hidden_size, input_ids, segment_ids, beta, gamma, + word_embedding, position_embedding, segment_embedding, + epsilon, output, embedding_sum, position_ids); - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } -bool LaunchEmbedLayerNormKernel( +Status LaunchEmbedLayerNormKernel( cudaStream_t stream, void* output, void* mask_index, @@ -203,10 +209,10 @@ bool LaunchEmbedLayerNormKernel( void* embedding_sum, const int* position_ids) { if (nullptr == input_mask) { - if (!CUDA_CALL(cudaMemsetAsync(mask_index, 0, sizeof(int) * batch_size, stream))) - return false; - } else if (!ComputeMaskIndex(stream, sequence_length, batch_size, input_mask, static_cast(mask_index))) { - return false; + CUDA_RETURN_IF_ERROR(cudaMemsetAsync(mask_index, 0, sizeof(int) * batch_size, stream)); + } else { + ORT_RETURN_IF_ERROR( + ComputeMaskIndex(stream, sequence_length, batch_size, input_mask, static_cast(mask_index))); } if (element_size == 2) { diff --git a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.h b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.h index 2358bef875..98268a9b0d 100644 --- a/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.h +++ b/onnxruntime/contrib_ops/cuda/bert/embed_layer_norm_impl.h @@ -1,29 +1,30 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. #pragma once +#include "core/common/common.h" namespace onnxruntime { namespace contrib { namespace cuda { -bool LaunchEmbedLayerNormKernel(cudaStream_t stream, - void* output, // output tensor - void* mask_index, // output mask index - const int* input_ids, // input word IDs - const int* segment_ids, // input segment IDs - const int* input_mask, // input mask - const void* gamma, // weight for layer normalization - const void* beta, // bias for layer normalization - const void* word_embedding, // weights for word embeddings - const void* position_embedding, // weights for position embeddings - const void* segment_embedding, // weights for segment (like sentence) embeddings - float epsilon, // epsilon for layer normalization - const int hidden_size, // hidden size (that is head_size * num_heads) - int batch_size, // batch size - int sequence_length, // sequence length - const size_t element_size, // size of element in output tensor. 2 for half, 4 for float. - void* embedding_sum, // Optional output of sum of embeddings - const int* position_ids); // Optional input of position ids +Status LaunchEmbedLayerNormKernel(cudaStream_t stream, + void* output, // output tensor + void* mask_index, // output mask index + const int* input_ids, // input word IDs + const int* segment_ids, // input segment IDs + const int* input_mask, // input mask + const void* gamma, // weight for layer normalization + const void* beta, // bias for layer normalization + const void* word_embedding, // weights for word embeddings + const void* position_embedding, // weights for position embeddings + const void* segment_embedding, // weights for segment (like sentence) embeddings + float epsilon, // epsilon for layer normalization + const int hidden_size, // hidden size (that is head_size * num_heads) + int batch_size, // batch size + int sequence_length, // sequence length + const size_t element_size, // size of output element: 2 for half, 4 for float. + void* embedding_sum, // Optional output of sum of embeddings + const int* position_ids); // Optional input of position ids } // namespace cuda } // namespace contrib diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc index 3b7b47bed3..30f242afe3 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc @@ -50,21 +50,16 @@ Status FastGelu::ComputeInternal(OpKernelContext* context) const { int64_t bias_length = (nullptr == bias) ? 0 : bias->Shape().Size(); typedef typename ToCudaType::MappedType CudaT; - if (!LaunchFastGeluKernel(GetDeviceProp(), + return LaunchFastGeluKernel(GetDeviceProp(), Stream(), static_cast(input_length), static_cast(bias_length), reinterpret_cast(input->Data()), (nullptr != bias) ? reinterpret_cast(bias->Data()) : nullptr, reinterpret_cast(output->MutableData()), - use_half2_)) { - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + use_half2_); } -} //namespace cuda +} // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h index 8155ece5b1..3e642a70af 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h @@ -16,7 +16,7 @@ class FastGelu final : public CudaKernel { public: FastGelu(const OpKernelInfo& op_kernel_info); Status ComputeInternal(OpKernelContext* ctx) const override; - + private: bool use_half2_; }; diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.cu b/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.cu index 8bfb7972d4..1b0de47a83 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.cu @@ -1,7 +1,7 @@ /* The implementation of this file is based on gelu plugin in TensorRT demo: https://github.com/NVIDIA/TensorRT/tree/release/5.1/demo/BERT/ - + Copyright 2019 NVIDIA Corporation Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,7 +40,8 @@ constexpr float B = 0.7978845608028654f; // sqrt(2.0/M_PI) constexpr float C = 0.035677408136300125f; // 0.044715 * sqrt(2.0/M_PI) template -__global__ void FastGeluKernel(const T a, const T b, const T c, int input_length, int bias_length, const T* input, const T* bias, T* output) { +__global__ void FastGeluKernel(const T a, const T b, const T c, int input_length, int bias_length, + const T* input, const T* bias, T* output) { const int idx = blockIdx.x * TPB + threadIdx.x; if (idx < input_length) { @@ -52,7 +53,8 @@ __global__ void FastGeluKernel(const T a, const T b, const T c, int input_length } template -__global__ void FastGeluKernel2(const half2 a, const half2 b, const half2 c, int input_length, int bias_length, const half2* input, const half2* bias, half2* output) { +__global__ void FastGeluKernel2(const half2 a, const half2 b, const half2 c, int input_length, int bias_length, + const half2* input, const half2* bias, half2* output) { const int idx = blockIdx.x * TPB + threadIdx.x; if (idx < input_length) { const half2 x = input[idx]; @@ -63,16 +65,19 @@ __global__ void FastGeluKernel2(const half2 a, const half2 b, const half2 c, int } template <> -bool LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, const float* input, const float* bias, float* output, bool /*use_half2*/) { +Status LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, + const float* input, const float* bias, float* output, bool /*use_half2*/) { constexpr int blockSize = 256; const int gridSize = (input_length + blockSize - 1) / blockSize; - FastGeluKernel<<>>(A, B, C, input_length, bias_length, input, bias, output); + FastGeluKernel<<>>(A, B, C, input_length, bias_length, + input, bias, output); - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } template <> -bool LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, const half* input, const half* bias, half* output, bool use_half2) { +Status LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, + const half* input, const half* bias, half* output, bool use_half2) { constexpr int blockSize = 256; if (use_half2 && 0 == (bias_length & 1) && prop.major >= 7) { const int n = input_length / 2; @@ -83,27 +88,29 @@ bool LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int i const half2* input2 = reinterpret_cast(input); const half2* bias2 = reinterpret_cast(bias); half2* output2 = reinterpret_cast(output); - FastGeluKernel2<<>>(A2, B2, C2, n, bias_length / 2, input2, bias2, output2); + FastGeluKernel2<<>>(A2, B2, C2, n, bias_length / 2, + input2, bias2, output2); } else { const int gridSize = (input_length + blockSize - 1) / blockSize; - FastGeluKernel<<>>(A, B, C, input_length, bias_length, input, bias, output); + FastGeluKernel<<>>(A, B, C, input_length, bias_length, + input, bias, output); } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } template <> -bool LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, +Status LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, const BFloat16* input, const BFloat16* bias, BFloat16* output, bool /*use_half2*/) { constexpr int blockSize = 256; // remove nv_bfloat162 implementation for now to fix build issue - // we can decide whether to add it back if there's perf concern + // we can decide whether to add it back if there's perf concern const int gridSize = (input_length + blockSize - 1) / blockSize; FastGeluKernel <<>>(A, B, C, input_length, bias_length, input, bias, output); - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.h b/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.h index 9618c52bb3..c708b38f2b 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.h +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu_impl.h @@ -2,13 +2,15 @@ // Licensed under the MIT License. #pragma once +#include "core/common/common.h" namespace onnxruntime { namespace contrib { namespace cuda { template -bool LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, const T* input, const T* bias, T* output, bool use_half2); +Status LaunchFastGeluKernel(const cudaDeviceProp& prop, cudaStream_t stream, int input_length, int bias_length, + const T* input, const T* bias, T* output, bool use_half2); } // namespace cuda } // namespace contrib diff --git a/onnxruntime/contrib_ops/cuda/bert/layer_norm.cuh b/onnxruntime/contrib_ops/cuda/bert/layer_norm.cuh index 6d29e67e66..0f1dbe5ab6 100644 --- a/onnxruntime/contrib_ops/cuda/bert/layer_norm.cuh +++ b/onnxruntime/contrib_ops/cuda/bert/layer_norm.cuh @@ -61,18 +61,21 @@ __device__ inline half2 AddHalf2(const half2 a, const half2 b) { } struct KeyValuePairSum { - __device__ inline cub::KeyValuePair operator()(const cub::KeyValuePair& a, const cub::KeyValuePair& b) { + __device__ inline cub::KeyValuePair operator()(const cub::KeyValuePair& a, + const cub::KeyValuePair& b) { return cub::KeyValuePair(a.key + b.key, a.value + b.value); } - __device__ inline cub::KeyValuePair operator()(const cub::KeyValuePair& a, const cub::KeyValuePair& b) { + __device__ inline cub::KeyValuePair operator()(const cub::KeyValuePair& a, + const cub::KeyValuePair& b) { const half2 a2 = __halves2half2(a.key, a.value); const half2 b2 = __halves2half2(b.key, b.value); const half2 res = AddHalf2(a2, b2); return cub::KeyValuePair(__low2half(res), __high2half(res)); } - __device__ inline cub::KeyValuePair operator()(const cub::KeyValuePair& a, const cub::KeyValuePair& b) { + __device__ inline cub::KeyValuePair operator()(const cub::KeyValuePair& a, + const cub::KeyValuePair& b) { return cub::KeyValuePair(AddHalf2(a.key, b.key), AddHalf2(a.value, b.value)); } }; @@ -139,10 +142,11 @@ __device__ inline void LayerNormSmall(const T* input_v, const cub::KeyValuePair< __syncthreads(); if (ILP * threadIdx.x < ld) { - #pragma unroll +#pragma unroll for (int i = 0; i < ILP; i++) { - output_v[i] = (beta != nullptr) ? gamma_v[i] * (input_v[i] - mu) * rsigma + beta_v[i] : - gamma_v[i] * (input_v[i] - mu) * rsigma; + output_v[i] = (beta != nullptr) + ? gamma_v[i] * (input_v[i] - mu) * rsigma + beta_v[i] + : gamma_v[i] * (input_v[i] - mu) * rsigma; } *(reinterpret_cast(&output[idx])) = *output_val; } diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc b/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc index 004fd98e08..4a0155705b 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention.cc @@ -3,11 +3,10 @@ #include "core/providers/cuda/shared_inc/fpgeneric.h" #include "core/platform/env_var_utils.h" -#include "contrib_ops/cuda/bert/longformer_attention.h" #include "contrib_ops/cuda/bert/longformer_global_impl.h" #include "contrib_ops/cuda/bert/longformer_attention_impl.h" #include "contrib_ops/cuda/bert/transformer_cuda_common.h" -#include "contrib_ops/cuda/bert/transformer_common.h" +#include "contrib_ops/cuda/bert/longformer_attention.h" using namespace onnxruntime::cuda; using namespace ::onnxruntime::common; @@ -33,8 +32,9 @@ REGISTER_KERNEL_TYPED(MLFloat16) template LongformerAttention::LongformerAttention(const OpKernelInfo& info) - : CudaKernel(info), LongformerAttentionBase(info) { + : CudaKernel(info), LongformerAttentionBase(info) { use_compact_memory_ = ParseEnvironmentVariableWithDefault(longformer::kUseCompactMemory, true); + use_half4_ = ParseEnvironmentVariableWithDefault(longformer::kUseHalf4, true); } template @@ -42,16 +42,27 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { const Tensor* input = context->Input(0); const Tensor* weights = context->Input(1); const Tensor* bias = context->Input(2); - const Tensor* mask = context->Input(3); + const Tensor* attention_mask = context->Input(3); const Tensor* global_weights = context->Input(4); const Tensor* global_bias = context->Input(5); - const Tensor* global_attention = context->Input(6); - ORT_RETURN_IF_ERROR(CheckInputs(input->Shape(), weights->Shape(), bias->Shape(), mask->Shape(), - global_weights->Shape(), global_bias->Shape(), global_attention->Shape())); + const Tensor* global_attention_mask = context->Input(6); + ORT_RETURN_IF_ERROR(CheckInputs(input->Shape(), weights->Shape(), bias->Shape(), attention_mask->Shape(), + global_weights->Shape(), global_bias->Shape(), global_attention_mask->Shape())); + // Input shapes: + // input : (batch_size, sequence_length, hidden_size) + // weights : (hidden_size, 3 * hidden_size) -- format 1 + // (3, hidden_size, hidden_size) -- format 0 + // bias : (3 * hidden_size) -- format 1 (bias for Q, K, V) + // (5 * hidden_size) -- format 0 (bias for Q, K, V, Global_K, Global_V) + // attention_mask : (batch_size, sequence_length) + // global_weights : (hidden_size, 3 * hidden_size) -- format 1 + // (3, hidden_size, hidden_size) -- format 0 + // global_bias : (3 * hidden_size) -- format 1 (bias for Global_Q, Global_K, Global_V) + // (1 * hidden_size) -- format 0 (bias for Global_Q) + // global_attention_mask : (batch_size, sequence_length) + // Output shapes: + // output : (batch_size, sequence_length, hidden_size) - // Input and output shapes: - // Input 0 - input : (batch_size, sequence_length, hidden_size) - // Output 0 - output : (batch_size, sequence_length, hidden_size) const auto& shape = input->Shape(); int batch_size = static_cast(shape[0]); int sequence_length = static_cast(shape[1]); @@ -66,23 +77,25 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { constexpr size_t element_size = sizeof(T); - // TODO: only calculate once per model. + // TODO(tianleiwu): only calculate global index once per model instead of once per LongformerAttention node. // Build Global Index auto global_index_buffer = GetScratchBuffer(batch_size * sequence_length); auto batch_global_num_buffer = GetScratchBuffer(batch_size); - size_t global_scratch_bytes = GetGlobalScratchSize(batch_size, sequence_length); + size_t global_scratch_bytes = GetGlobalScratchSize(sequence_length); auto global_scratch_buffer = GetScratchBuffer(global_scratch_bytes); - BuildGlobalIndex( + auto& device_prop = GetDeviceProp(); + ORT_RETURN_IF_ERROR(BuildGlobalIndex( + device_prop, stream, - global_attention->Data(), + global_attention_mask->Data(), batch_size, sequence_length, global_index_buffer.get(), batch_global_num_buffer.get(), global_scratch_buffer.get(), - global_scratch_bytes); + global_scratch_bytes)); // Copy batch_global_num to CPU size_t pinned_buffer_bytes = GetPinnedBufferSize(batch_size); @@ -96,40 +109,66 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { // Create an event to make sure the async copy is finished before reading the data. AutoDestoryCudaEvent new_event; - cudaEvent_t& isCopyDone = new_event.Get(); + cudaEvent_t& is_copy_done = new_event.Get(); - CUDA_RETURN_IF_ERROR(cudaEventCreate(&isCopyDone)); - CUDA_RETURN_IF_ERROR(cudaEventRecord(isCopyDone, stream)); - - // Use GEMM for fully connection. - int m = batch_size * sequence_length; - int n = 3 * hidden_size; - int k = hidden_size; + CUDA_RETURN_IF_ERROR(cudaEventCreateWithFlags(&is_copy_done, cudaEventDisableTiming)); + CUDA_RETURN_IF_ERROR(cudaEventRecord(is_copy_done, stream)); size_t qkv_size = batch_size * sequence_length * 3 * hidden_size * element_size; - auto gemm_buffer = GetScratchBuffer(qkv_size); + // Buffer for GEMM outputs of q, k, v, global_q, global_k and global_v + // TODO(tianleiwu): compact global_q only need batch_size * window * hidden_size * element_size buffer size. + auto gemm_buffer = GetScratchBuffer(qkv_size + qkv_size); + bool use_merged_qkv_weights = (weights->Shape().NumDimensions() == 2); + + int m = batch_size * sequence_length; + int n = use_merged_qkv_weights ? 3 * hidden_size : hidden_size; + int k = hidden_size; typedef typename ToCudaType::MappedType CudaT; - CudaT one = ToCudaType::FromFloat(1.0f); - CudaT zero = ToCudaType::FromFloat(0.0f); + const CudaT* input_data = reinterpret_cast(input->Data()); + const CudaT* weights_data = reinterpret_cast(weights->Data()); + const CudaT* global_weights_data = reinterpret_cast(global_weights->Data()); - // Bias shape is (N), broadcast using B(N, M) = 1 * bias(N, 1) x ones(1, M) + 0 * B. - auto& device_prop = GetDeviceProp(); - CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, 1, &one, - reinterpret_cast(bias->Data()), n, - GetConstOnes(m), 1, - &zero, reinterpret_cast(gemm_buffer.get()), n, device_prop)); + float one = 1.0f; + float zero = 0.0f; + if (use_merged_qkv_weights) { + // Gemm, note that CUDA assumes col-major, so result(N, M) = 1 * weights x input + 0 x B. + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + weights_data, n, + input_data, k, + &zero, reinterpret_cast(gemm_buffer.get()), n, device_prop)); + } else { + // q + const CudaT* q_weight = weights_data; + CudaT* q_data = reinterpret_cast(gemm_buffer.get()); + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + q_weight, n, + input_data, k, + &zero, q_data, n, device_prop)); - // Gemm, note that CUDA assumes col-major, so result(N, M) = 1 * weights x input + 1 x B. - CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, - reinterpret_cast(weights->Data()), n, - reinterpret_cast(input->Data()), k, - &one, reinterpret_cast(gemm_buffer.get()), n, device_prop)); + // k + const CudaT* k_weight = q_weight + hidden_size * hidden_size; + CudaT* k_data = q_data + batch_size * sequence_length * hidden_size; + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + k_weight, n, + input_data, k, + &zero, k_data, n, device_prop)); + + // v + const CudaT* v_weight = k_weight + hidden_size * hidden_size; + CudaT* v_data = k_data + batch_size * sequence_length * hidden_size; + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + v_weight, n, + input_data, k, + &zero, v_data, n, device_prop)); + } // Wait for async copy of batch_global_num - CUDA_RETURN_IF_ERROR(cudaEventSynchronize(isCopyDone)); + CUDA_RETURN_IF_ERROR(cudaEventSynchronize(is_copy_done)); // Find the maximum number of global tokens in all batches int max_num_global = 0; @@ -139,7 +178,7 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { } } - // Do not use compact kernel in the following situations: + // Do not use compact memory kernel in the following situations: // (1) global tokens > windows size, compact memory kernel cannot be used due to its assumptions. // (2) sequence_length == 2 * attention_window, compact memory kernel has parity issue. // (3) user sets environment variable ORT_LONGFORMER_COMPACT_MEMORY=0 @@ -147,21 +186,67 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { // Fully connection for global projection. // Note that Q only need handle global query tokens if we split GEMM to global Q/K/V separately. - // When there is no global token, need not run glboal GEMM. - auto global_gemm_buffer = GetScratchBuffer(max_num_global > 0 ? qkv_size : 0); + // When there is no global token, need not run global GEMM. + CudaT* global_gemm_buffer = nullptr; if (max_num_global > 0) { - CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, 1, &one, - reinterpret_cast(global_bias->Data()), n, - GetConstOnes(m), 1, - &zero, reinterpret_cast(global_gemm_buffer.get()), n, device_prop)); + global_gemm_buffer = reinterpret_cast(reinterpret_cast(gemm_buffer.get()) + qkv_size); - CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( - cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, - reinterpret_cast(global_weights->Data()), n, - reinterpret_cast(input->Data()), k, - &one, reinterpret_cast(global_gemm_buffer.get()), n, device_prop)); + if (use_merged_qkv_weights) { + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + reinterpret_cast(global_weights->Data()), n, + input_data, k, + &zero, global_gemm_buffer, n, device_prop)); + } else { + // global q + const CudaT* global_q_weight = global_weights_data; + CudaT* global_q = global_gemm_buffer + 2 * batch_size * sequence_length * hidden_size; + if (disable_compact_memory) { + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + global_q_weight, n, + input_data, k, + &zero, global_q, n, device_prop)); + } else { + CUBLAS_RETURN_IF_ERROR(cublasGemmStridedBatchedHelper(cublas, + CUBLAS_OP_N, + CUBLAS_OP_N, + hidden_size, // m + max_num_global, // n + hidden_size, // k + &one, // alpha + global_q_weight, // A + hidden_size, // lda + 0, // strideA + input_data, // B + hidden_size, // ldb + sequence_length * hidden_size, // strideB + &zero, // beta + global_q, // C + hidden_size, // ldc + max_num_global * hidden_size, // strideC + batch_size, // batch count + device_prop)); + } + // global k + const CudaT* global_k_weight = global_weights_data + hidden_size * hidden_size; + CudaT* global_k = global_gemm_buffer; + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + global_k_weight, n, + input_data, k, + &zero, global_k, n, device_prop)); + + // global v + const CudaT* global_v_weight = global_k_weight + hidden_size * hidden_size; + CudaT* global_v = global_gemm_buffer + batch_size * sequence_length * hidden_size; + CUBLAS_RETURN_IF_ERROR(cublasGemmHelper( + cublas, CUBLAS_OP_N, CUBLAS_OP_N, n, m, k, &one, + global_v_weight, n, + input_data, k, + &zero, global_v, n, device_prop)); + } } size_t workSpaceSize = GetLongformerAttentionWorkspaceSize(element_size, @@ -173,14 +258,16 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { window_, disable_compact_memory); auto workspace_buffer = GetScratchBuffer(workSpaceSize); - if (!LaunchLongformerAttentionKernel( + ORT_RETURN_IF_ERROR(LaunchLongformerAttentionKernel( device_prop, cublas, stream, reinterpret_cast(gemm_buffer.get()), - reinterpret_cast(mask->Data()), - reinterpret_cast(global_gemm_buffer.get()), - global_attention->Data(), + reinterpret_cast(bias->Data()), + reinterpret_cast(attention_mask->Data()), + reinterpret_cast(global_gemm_buffer), + reinterpret_cast(global_bias->Data()), + global_attention_mask->Data(), global_index_buffer.get(), batch_global_num_buffer.get(), pinned_buffer.get(), @@ -193,13 +280,11 @@ Status LongformerAttention::ComputeInternal(OpKernelContext* context) const { window_, max_num_global, element_size, - disable_compact_memory)) { - // Get last error to reset it to cudaSuccess. - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } + disable_compact_memory, + use_merged_qkv_weights, + use_half4_)); - // Defer release of pinnned memory since cudaStreamSynchronize is not used here and kernel need access the buffer. + // Defer release of pinned memory since cudaStreamSynchronize is not used here and kernel need access the buffer. this->AddDeferredReleaseCPUPtr(pinned_buffer.release()); return Status::OK(); diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention.h b/onnxruntime/contrib_ops/cuda/bert/longformer_attention.h index fa9dfd8c9b..aabb21466f 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention.h +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention.h @@ -21,6 +21,7 @@ class LongformerAttention final : public CudaKernel, public LongformerAttentionB private: bool use_compact_memory_; + bool use_half4_; }; } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.cu index bf4c23328b..7d2a8600ef 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.cu @@ -26,23 +26,16 @@ limitations under the License. #include #include "core/providers/cuda/cu_inc/common.cuh" #include "core/providers/cuda/cuda_common.h" -#include "contrib_ops/cuda/bert/longformer_attention_impl.h" +#include "contrib_ops/cuda/bert/add_bias_transpose.h" #include "contrib_ops/cuda/bert/attention_impl.h" #include "contrib_ops/cuda/bert/longformer_attention_softmax.h" -#include "core/common/safeint.h" +#include "contrib_ops/cuda/bert/longformer_attention_impl.h" using namespace onnxruntime::cuda; using namespace cub; -#define CHECK(expr) \ - if (!CUBLAS_CALL(expr)) { \ - return false; \ - } - -#define CHECK_CUDA(expr) \ - if (!CUDA_CALL(expr)) { \ - return false; \ - } +#define CHECK(expr) CUBLAS_RETURN_IF_ERROR(expr) +#define CHECK_CUDA(expr) CUDA_RETURN_IF_ERROR(expr) namespace onnxruntime { namespace contrib { @@ -55,45 +48,51 @@ namespace cuda { // where Global_Q, Global_K and Global_V are optional. They are not allocated when there is no global token. // // SoftmaxSpace layout is the following when compact memory is enabled: -// [scratch1: (5S-3W)*W*N*B] [scratch2: size_t 20] +// [scratch1: (5S-3W)*W*N*B] [scratch2: size_t 15] // Scratch1 has 5 buffers for local and global attention calculation. -// Scratch2 has 5 input pointers, 5 output pointers, 5 buffer sizes and 5 strides related to scratch1. +// Scratch2 has 5 input/output pointers, 5 buffer sizes and 5 strides related to scratch1. // // SoftmaxSpace layout is the following When compact memory is disabled: // [scratch1: BxNxSxS] [scratch2: BxNxSxS] -size_t GetScratch1Size(size_t element_size, int batch_size, int num_heads, int sequence_length, int window) { - return SafeInt(5 * sequence_length - 3 * window) * window * num_heads * batch_size * element_size; +static size_t Align(size_t a) { + const size_t alignment = 128; // Align on a 16-byte boundary to avoid "misaligned address" error. + return CeilDiv(a, alignment) * alignment; +} + +size_t GetScratch1Size(size_t element_size, size_t batch_size, size_t num_heads, size_t sequence_length, size_t window) { + size_t bytes = (5 * sequence_length - 3 * window) * window * num_heads * batch_size * element_size; + return Align(bytes); } constexpr size_t GetScratch2Size() { - return 10 * (sizeof(void*) + sizeof(size_t)); + return 5 * sizeof(void*) + 10 * sizeof(size_t); } size_t GetLongformerSoftmaxWorkspaceSize( size_t element_size, - int batch_size, - int num_heads, - int sequence_length, - int window, + size_t batch_size, + size_t num_heads, + size_t sequence_length, + size_t window, bool disable_compact_memory) { if (!disable_compact_memory) { size_t scratch1_size = GetScratch1Size(element_size, batch_size, num_heads, sequence_length, window); - size_t scratch2_size = 10 * (sizeof(void*) + sizeof(size_t)); - return scratch1_size + scratch2_size; + size_t scratch2_size = GetScratch2Size(); + return Align(scratch1_size + scratch2_size); } else { - return SafeInt(2) * GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, sequence_length); + return 2 * GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, sequence_length); } } size_t GetLongformerAttentionWorkspaceSize( size_t element_size, - int batch_size, - int num_heads, - int head_size, - int sequence_length, - int max_num_global, - int window, + size_t batch_size, + size_t num_heads, + size_t head_size, + size_t sequence_length, + size_t max_num_global, + size_t window, bool disable_compact_memory) { size_t softmax_size = GetLongformerSoftmaxWorkspaceSize(element_size, batch_size, @@ -101,14 +100,14 @@ size_t GetLongformerAttentionWorkspaceSize( sequence_length, window, disable_compact_memory); - size_t qkv_size = SafeInt(3) * batch_size * sequence_length * num_heads * head_size * element_size; + size_t qkv_size = static_cast(3) * batch_size * sequence_length * num_heads * head_size * element_size; size_t global_qkv_size = max_num_global > 0 ? qkv_size : 0; return softmax_size + qkv_size + global_qkv_size; } // Size of buffer of pinned memory in CPU. The buffer is used to copy memory between CPU and GPU. // The buffer includes two parts: [global_count (copy of batch_global_num): int Bx1] [copy of scratch2] -size_t GetPinnedBufferSize(int batch_size) { +size_t GetPinnedBufferSize(size_t batch_size) { return sizeof(int) * batch_size + GetScratch2Size(); } @@ -118,20 +117,17 @@ __launch_bounds__(blockSize) __global__ void LongformerSoftmaxKernel(const int* global_attention, const int* global_index, const int* batch_global_num, - void* input_pointers, + void* buffer_pointers, const T* attention_mask, float scaler, - int dim0, int sequence_length, - int window, - int num_heads) { + int num_heads, + int window) { typedef cub::BlockReduce BlockReduce; __shared__ typename BlockReduce::TempStorage block_reduce_temp; - __shared__ float max_shared; - __shared__ float sum_shared; int tid = threadIdx.x; - const int batch_index = blockIdx.x / dim0; + const int batch_index = blockIdx.x / (sequence_length * num_heads); const int row_index = blockIdx.x % sequence_length; const int head_index = (blockIdx.x / sequence_length) % num_heads; @@ -139,17 +135,15 @@ __launch_bounds__(blockSize) const T* mask_block = attention_mask + sequence_length * batch_index; const int* global_index_block = global_index + sequence_length * batch_index; const int global_num = batch_global_num[batch_index]; - - size_t* p_inputs = (size_t*)(input_pointers); - size_t* p_outputs = (size_t*)(input_pointers) + 5; - size_t* input_sizes = (size_t*)(input_pointers) + 10; - size_t* input_strides = (size_t*)(input_pointers) + 15; - + size_t* p_inputs = reinterpret_cast(buffer_pointers); + size_t* p_outputs = reinterpret_cast(buffer_pointers); + size_t* input_sizes = reinterpret_cast(buffer_pointers) + 5; + size_t* input_strides = reinterpret_cast(buffer_pointers) + 10; const T* inputs[5]; T* outputs[5]; for (int i = 0; i < 5; ++i) { - inputs[i] = (T*)p_inputs[i] + batch_index * num_heads * input_sizes[i]; - outputs[i] = (T*)p_outputs[i] + batch_index * num_heads * input_sizes[i]; + inputs[i] = reinterpret_cast(p_inputs[i]) + batch_index * num_heads * input_sizes[i]; + outputs[i] = reinterpret_cast(p_outputs[i]) + batch_index * num_heads * input_sizes[i]; } // Local attention token @@ -211,9 +205,11 @@ __launch_bounds__(blockSize) } float sum_input = 0.; + __shared__ float sum_shared; // Calculate max input float max_input = -CUDART_INF_F; + __shared__ float max_shared; if (is_local_row) { const T* input_block = nullptr; @@ -245,7 +241,9 @@ __launch_bounds__(blockSize) const T* input_global = nullptr; int local_global = row_index - window; - if (local_global > global_num) local_global = global_num; + if (local_global > global_num) { + local_global = global_num; + } if (local_global > 0) { input_global = inputs[3] + (row_index - window) * input_strides[3] + head_index * input_sizes[3]; } @@ -363,7 +361,7 @@ __launch_bounds__(blockSize) } } -bool LaunchLongformerSoftmaxKernel( +Status LaunchLongformerSoftmaxKernel( cudaStream_t stream, cublasHandle_t cublas, void* workspace, @@ -371,6 +369,8 @@ bool LaunchLongformerSoftmaxKernel( const void* k, // transposed K with shape (B, N, S, H) const void* v, // transposed V with shape (B, N, S, H) const void* attention_mask, // attention mask with shape (B, S), with value 0 not masked and -10000 masked. + int max_num_global, // maximum number of global tokens (G) + const bool compact_global_q, // whether global_q has shape (B, N, G, H) instead of (B, N, S, H) const void* global_q, // Q for global tokens with shape (B, N, S, H). const void* global_k, // K for global tokens with shape (B, N, S, H) const void* global_v, // V for global tokens with shape (B, N, S, H) @@ -430,7 +430,7 @@ bool LaunchLongformerSoftmaxKernel( // qk = q * k^T // Shapes: q and k = B x N x S x H, qk = B x N x S x S // Convert col-major to row-major by swapping q and k in Gemm - int elements_per_batch = num_heads * sequence_length * head_size; + size_t elements_per_batch = num_heads * sequence_length * head_size; int stride_per_head = sequence_length * head_size; // stride for Q, K, V and output // Local attention part @@ -440,7 +440,7 @@ bool LaunchLongformerSoftmaxKernel( // [W][W][W] // [W][W] // The first and last rows have 2 blocks per row, and the remaining has 3 blocks per row. - // The calculation are splited into 3 parts: middle rows, then the first row and finally the last row. + // The calculation are splited into 3 parts: the first row, middle rows and finally the last row. // To save space, we do not store the whole matrix. Instead, we only allocate space for these blocks. // // For global attention part, we have two assumptions: @@ -474,38 +474,59 @@ bool LaunchLongformerSoftmaxKernel( static_cast(w), // number of global tokens <= window size static_cast(sequence_length)}; - void* input_pointers[5]; - void* output_pointers[5]; + void* buffer_pointers[5]; char* current_pointer = scratch1; for (int i = 0; i < 5; ++i) { - input_pointers[i] = (void*)current_pointer; - output_pointers[i] = (void*)current_pointer; // output pointer is same as input + buffer_pointers[i] = reinterpret_cast(current_pointer); current_pointer += buffer_sizes[i] * num_heads * batch_size * element_size; } - assert(current_pointer == scratch2); // Copy to a continues buffer first so that we only need call cudaMemcpyAsync once - constexpr size_t totalBytes = 10 * (sizeof(size_t) + sizeof(void*)); char* temp_buffer = reinterpret_cast(pinned_buffer) + sizeof(int) * batch_size; - memcpy(temp_buffer, &input_pointers[0], 5 * sizeof(void*)); - memcpy(temp_buffer + 5 * sizeof(void*), &output_pointers[0], 5 * sizeof(void*)); - memcpy(temp_buffer + 10 * sizeof(void*), &buffer_sizes[0], 5 * sizeof(size_t)); - memcpy(temp_buffer + 10 * sizeof(void*) + 5 * sizeof(size_t), &buffer_strides[0], 5 * sizeof(size_t)); - CHECK_CUDA(cudaMemcpyAsync(scratch2, temp_buffer, totalBytes, cudaMemcpyHostToDevice, stream)); + memcpy(temp_buffer, &buffer_pointers[0], 5 * sizeof(void*)); + memcpy(temp_buffer + 5 * sizeof(void*), &buffer_sizes[0], 5 * sizeof(size_t)); + memcpy(temp_buffer + 5 * sizeof(void*) + 5 * sizeof(size_t), &buffer_strides[0], 5 * sizeof(size_t)); + CHECK_CUDA(cudaMemcpyAsync(scratch2, temp_buffer, GetScratch2Size(), cudaMemcpyHostToDevice, stream)); // Local attention part { + // local attention per head - head + CHECK(cublasGemmStridedBatchedEx(cublas, + CUBLAS_OP_T, + CUBLAS_OP_N, + 2 * w, // m + w, // n + head_size, // k + alpha, // alpha + k, // A + Atype, // A type + head_size, // lda + stride_per_head, // strideA + q, // B + Btype, // B type + head_size, // ldb + stride_per_head, // strideB + beta_0, // beta + buffer_pointers[0], // C + Ctype, // C type + 2 * w, // ldc + buffer_sizes[0], // strideC + batch_size * num_heads, // batch count + resultType, + algo)); + + // local attention per head - middle if (middle_count > 0) { for (int i = 0; i < batch_size; ++i) { for (int j = 0; j < num_heads; ++j) { - const void* q_head = reinterpret_cast(q) + \ - (i * elements_per_batch + j * sequence_length * head_size + w * head_size) * element_size; - const void* k_head = reinterpret_cast(k) + \ + const void* q_head = reinterpret_cast(q) + + (i * elements_per_batch + (j * sequence_length + w) * head_size) * element_size; + const void* k_head = reinterpret_cast(k) + (i * elements_per_batch + j * sequence_length * head_size) * element_size; - void* qk_head = reinterpret_cast(input_pointers[1]) + \ - (i * num_heads + j) * buffer_sizes[1] * element_size; + void* qk_head = reinterpret_cast(buffer_pointers[1]) + + static_cast(i * num_heads + j) * buffer_sizes[1] * element_size; CHECK(cublasGemmStridedBatchedEx(cublas, CUBLAS_OP_T, CUBLAS_OP_N, @@ -533,30 +554,7 @@ bool LaunchLongformerSoftmaxKernel( } } - CHECK(cublasGemmStridedBatchedEx(cublas, - CUBLAS_OP_T, - CUBLAS_OP_N, - 2 * w, // m - w, // n - head_size, // k - alpha, // alpha - k, // A - Atype, // A type - head_size, // lda - stride_per_head, // strideA - q, // B - Btype, // B type - head_size, // ldb - stride_per_head, // strideB - beta_0, // beta - input_pointers[0], // C - Ctype, // C type - 2 * w, // ldc - buffer_sizes[0], // strideC - batch_size * num_heads, // batch count - resultType, - algo)); - + // local attention per head - tail const void* q_head = reinterpret_cast(q) + (last_block * w * head_size) * element_size; const void* k_head = reinterpret_cast(k) + ((last_block - 1) * w * head_size) * element_size; @@ -576,7 +574,7 @@ bool LaunchLongformerSoftmaxKernel( head_size, // ldb stride_per_head, // strideB beta_0, // beta - input_pointers[2], // C + buffer_pointers[2], // C Ctype, // C type 2 * w, // ldc buffer_sizes[2], // strideC @@ -590,7 +588,7 @@ bool LaunchLongformerSoftmaxKernel( if (global_count[i] > 0) { const void* q_batch = reinterpret_cast(q) + (i * elements_per_batch + w * head_size) * element_size; const void* k_batch = reinterpret_cast(k) + (i * elements_per_batch) * element_size; - void* qk_batch = reinterpret_cast(input_pointers[3]) + (i * buffer_sizes[3]) * num_heads * element_size; + void* qk_batch = reinterpret_cast(buffer_pointers[3]) + (i * buffer_sizes[3]) * num_heads * element_size; // Local tokens attending global tokens CHECK(cublasGemmStridedBatchedEx(cublas, @@ -617,13 +615,11 @@ bool LaunchLongformerSoftmaxKernel( resultType, algo)); - // It is feasible to use compact format for Global_Q with shape BxNxGxH to save space. - // In that case, elements_per_batch is num_heads * max_num_global * head_size, - // and stride_per_head is max_num_global * head_size. - - const void* global_q_batch = reinterpret_cast(global_q) + (i * elements_per_batch) * element_size; + const size_t global_q_per_batch = compact_global_q ? num_heads * max_num_global * head_size : elements_per_batch; + const int global_q_stride = (compact_global_q ? max_num_global * head_size : stride_per_head); + const void* global_q_batch = reinterpret_cast(global_q) + (i * global_q_per_batch) * element_size; const void* global_k_batch = reinterpret_cast(global_k) + (i * elements_per_batch) * element_size; - qk_batch = reinterpret_cast(input_pointers[4]) + (i * buffer_sizes[4] * num_heads) * element_size; + qk_batch = reinterpret_cast(buffer_pointers[4]) + (i * buffer_sizes[4] * num_heads) * element_size; // Global tokens attending everything // This GEMMs need to be last to make sure all global token entries are re-written. @@ -641,7 +637,7 @@ bool LaunchLongformerSoftmaxKernel( global_q_batch, // B Btype, // B type head_size, // ldb - stride_per_head, // strideB. + global_q_stride, // strideB. beta_0, // beta qk_batch, // C Ctype, // C type @@ -653,9 +649,6 @@ bool LaunchLongformerSoftmaxKernel( } } - int dim0 = sequence_length * num_heads; - int dim1 = sequence_length; - const int blockSize = 64; const int gridSize = batch_size * num_heads * sequence_length; if (is_fp16) { @@ -665,7 +658,7 @@ bool LaunchLongformerSoftmaxKernel( batch_global_num, scratch2, static_cast(attention_mask), - scaler, dim0, dim1, window, num_heads); + scaler, sequence_length, num_heads, window); } else { LongformerSoftmaxKernel<<>>( global_attention, @@ -673,183 +666,196 @@ bool LaunchLongformerSoftmaxKernel( batch_global_num, scratch2, static_cast(attention_mask), - scaler, dim0, dim1, window, num_heads); + scaler, sequence_length, num_heads, window); } - // Run the matrix multiply: output = softmax_out * v - // softmax_out: B x N x S x S - // v: B x N x S x H - // attn_out: B x N x S x H - // Calculation uses sliding blocks in a way similar to local attention part. - + // local values attending the softmax score. { + // local attention per head - head + CHECK(cublasGemmStridedBatchedEx(cublas, + CUBLAS_OP_N, + CUBLAS_OP_N, + head_size, // m + w, // n + 2 * w, // k + alpha, // alpha + v, // A + Atype, // A type + head_size, // lda + stride_per_head, // strideA + buffer_pointers[0], // B + Btype, // B type + static_cast(buffer_strides[0]), // ldb + buffer_sizes[0], // strideB + beta_0, // beta + output, // C + Ctype, // C type + head_size, // ldc + stride_per_head, // strideC + batch_size * num_heads, // batch count + resultType, + algo)); + + // local attention per head - middle if (middle_count > 0) { for (int i = 0; i < batch_size; ++i) { for (int j = 0; j < num_heads; ++j) { - const void* v_head = reinterpret_cast(v) + \ + const void* v_head = reinterpret_cast(v) + (i * elements_per_batch + j * head_size * sequence_length) * element_size; - const void* prob_head = reinterpret_cast(output_pointers[1]) + \ - (i * num_heads * buffer_sizes[1] + j * buffer_sizes[1]) * element_size; - void* out_head = reinterpret_cast(output) + \ + const void* prob_head = reinterpret_cast(buffer_pointers[1]) + + (i * num_heads + j) * buffer_sizes[1] * element_size; + void* out_head = reinterpret_cast(output) + (i * elements_per_batch + j * head_size * sequence_length + w * head_size) * element_size; CHECK(cublasGemmStridedBatchedEx(cublas, CUBLAS_OP_N, CUBLAS_OP_N, - head_size, // m - w, // n - 3 * w, // k - alpha, // alpha - v_head, // A - Atype, // A type - head_size, // lda - w * head_size, // strideA - prob_head, // B - Btype, // B type - (int)buffer_strides[1], // ldb - 3 * w * w, // strideB - beta_0, // beta - out_head, // C - Ctype, // C type - head_size, // ldc - w * head_size, // strideC - middle_count, // batch count + head_size, // m + w, // n + 3 * w, // k + alpha, // alpha + v_head, // A + Atype, // A type + head_size, // lda + w * head_size, // strideA + prob_head, // B + Btype, // B type + static_cast(buffer_strides[1]), // ldb + 3 * w * w, // strideB + beta_0, // beta + out_head, // C + Ctype, // C type + head_size, // ldc + w * head_size, // strideC + middle_count, // batch count resultType, algo)); } } } - CHECK(cublasGemmStridedBatchedEx(cublas, - CUBLAS_OP_N, - CUBLAS_OP_N, - head_size, // m - w, // n - 2 * w, // k - alpha, // alpha - v, // A - Atype, // A type - head_size, // lda - stride_per_head, // strideA - output_pointers[0], // B - Btype, // B type - (int)buffer_strides[0], // ldb - buffer_sizes[0], // strideB - beta_0, // beta - output, // C - Ctype, // C type - head_size, // ldc - stride_per_head, // strideC - batch_size * num_heads, // batch count - resultType, - algo)); - + // local attention per head - tail const void* v_head = reinterpret_cast(v) + (last_block - 1) * w * head_size * element_size; void* out_head = reinterpret_cast(output) + last_block * w * head_size * element_size; CHECK(cublasGemmStridedBatchedEx(cublas, CUBLAS_OP_N, CUBLAS_OP_N, - head_size, // m - w, // n - 2 * w, // k - alpha, // alpha - v_head, // A - Atype, // A type - head_size, // lda - stride_per_head, // strideA - output_pointers[2], // B - Btype, // B type - (int)buffer_strides[2], // ldb - buffer_sizes[2], // strideB - beta_0, // beta - out_head, // C - Ctype, // C type - head_size, // ldc - stride_per_head, // strideC - batch_size * num_heads, // batch count + head_size, // m + w, // n + 2 * w, // k + alpha, // alpha + v_head, // A + Atype, // A type + head_size, // lda + stride_per_head, // strideA + buffer_pointers[2], // B + Btype, // B type + static_cast(buffer_strides[2]), // ldb + buffer_sizes[2], // strideB + beta_0, // beta + out_head, // C + Ctype, // C type + head_size, // ldc + stride_per_head, // strideC + batch_size * num_heads, // batch count resultType, algo)); } + // global attention part for (int i = 0; i < batch_size; ++i) { if (global_count[i] > 0) { - int glob_longdim_mm = sequence_length - 2 * w; - + // Local tokens attending global tokens const void* v_head = reinterpret_cast(v) + (i * elements_per_batch) * element_size; - const void* prob_head = reinterpret_cast(output_pointers[3]) + \ + const void* prob_head = reinterpret_cast(buffer_pointers[3]) + (i * buffer_sizes[3] * num_heads + w * buffer_strides[3]) * element_size; void* out_head = reinterpret_cast(output) + (i * elements_per_batch + 2 * w * head_size) * element_size; CHECK(cublasGemmStridedBatchedEx(cublas, CUBLAS_OP_N, CUBLAS_OP_N, - head_size, // m - glob_longdim_mm, // n - global_count[i], // k - alpha, // alpha - v_head, // A - Atype, // A type - head_size, // lda - stride_per_head, // strideA - prob_head, // B - Btype, // B type - (int)buffer_strides[3], // ldb - buffer_sizes[3], // strideB - beta_1, // beta - out_head, // C - Ctype, // C type - head_size, // ldc - stride_per_head, // strideC - num_heads, // batch count + head_size, // m + sequence_length - 2 * w, // n + global_count[i], // k + alpha, // alpha + v_head, // A + Atype, // A type + head_size, // lda + stride_per_head, // strideA + prob_head, // B + Btype, // B type + static_cast(buffer_strides[3]), // ldb + buffer_sizes[3], // strideB + beta_1, // beta + out_head, // C + Ctype, // C type + head_size, // ldc + stride_per_head, // strideC + num_heads, // batch count resultType, algo)); - // Global tokens + // Global tokens attending everything v_head = reinterpret_cast(global_v) + (i * elements_per_batch) * element_size; - prob_head = reinterpret_cast(output_pointers[4]) + (i * buffer_sizes[4] * num_heads) * element_size; + prob_head = reinterpret_cast(buffer_pointers[4]) + (i * buffer_sizes[4] * num_heads) * element_size; out_head = reinterpret_cast(output) + (i * elements_per_batch) * element_size; CHECK(cublasGemmStridedBatchedEx(cublas, CUBLAS_OP_N, CUBLAS_OP_N, - head_size, // m - global_count[i], // n - sequence_length, // k: re-write entries completely - alpha, // alpha - v_head, // A - Atype, // A type - head_size, // lda - stride_per_head, // strideA - prob_head, // B - Btype, // B type - (int)buffer_strides[4], // ldb - buffer_sizes[4], // strideB - beta_0, // beta: overwrite - out_head, // C: assumes global tokens at the beginning of sequence - Ctype, // C type - head_size, // ldc - stride_per_head, // strideC - num_heads, // batch count + head_size, // m + global_count[i], // n + sequence_length, // k: re-write entries completely + alpha, // alpha + v_head, // A + Atype, // A type + head_size, // lda + stride_per_head, // strideA + prob_head, // B + Btype, // B type + static_cast(buffer_strides[4]), // ldb + buffer_sizes[4], // strideB + beta_0, // beta: overwrite + out_head, // C: assumes global tokens at the beginning of sequence + Ctype, // C type + head_size, // ldc + stride_per_head, // strideC + num_heads, // batch count resultType, algo)); } } - return true; + return Status::OK(); } template -bool LongformerQkvToContext( - const cudaDeviceProp& device_prop, cublasHandle_t cublas, cudaStream_t stream, - const int batch_size, const int sequence_length, const int num_heads, const int head_size, - const int window, const size_t element_size, - const T* input, const T* attention_mask, - const T* global_input, const int* global_attention, - const int* global_index, const int* batch_global_num, const int max_num_global, - void* pinned_buffer, T* workspace, - T* output, +Status LongformerQkvToContext( + const cudaDeviceProp& device_prop, + cublasHandle_t cublas, + cudaStream_t stream, + const int batch_size, // batch size + const int sequence_length, // sequence length + const int num_heads, // number of attention heads + const int head_size, // hidden size per head + const int window, // Half (one-sided) window size + const size_t element_size, + const T* input, // input for transpose + const T* bias, // bias to add to transposed input + const T* attention_mask, // attention mask with shape (B, S), with value 0.0 not masked, and -10000.0 masked. + const T* global_input, // global input for transpose + const T* global_bias, // bias to add to transposed global input + const int* global_attention, // global attention flags with shape (B, S), with value 0 for local and 1 for global. + const int* global_index, // Global index with shape (B, S) + const int* batch_global_num, // Number of global tokens per batch with shape (B, 1) + const int max_num_global, // Maximum number of global tokens (G) + void* pinned_buffer, // Pinned memory in CPU. Number of global tokens per batch with shape (B, 1) + T* workspace, // Softmax space + T* output, // output size_t softmax_workspace_size, - bool disable_compact_memory) { + bool disable_compact_memory, + bool use_merged_qkv_weights, + bool use_half4) { T* qkv = reinterpret_cast(reinterpret_cast(workspace) + softmax_workspace_size); // Number of elements in Q, K, V, Global_Q, Global_K or Global_V are same: BxNxSxH @@ -857,47 +863,48 @@ bool LongformerQkvToContext( const int max_threads_per_block(device_prop.maxThreadsPerBlock); - // Input should be BxSx3xNxH => qkv: 3xBxNxSxH - if (!LaunchTransQkv(stream, - 3, - sequence_length, - batch_size, - head_size, - num_heads, - max_threads_per_block, - false, - input, - qkv)) { - return false; - } + const int format = static_cast(use_merged_qkv_weights); + bool compact_global_q = false; + // The order of qkv space: + // Q, K, V, Global_K, Global_V, Global_Q (format 0) + // Q, K, V, Global_Q, Global_K, Global_V (format 1) + if (format == 1 || max_num_global == 0 || nullptr == global_input) { + LaunchAddBiasTranspose(stream, 3, format, max_threads_per_block, batch_size, + sequence_length, num_heads, head_size, + input, bias, qkv, + use_half4); - // Input 'global_input' should be BxSx3xNxH => global_qkv: 3xBxNxSxH - T* global_qkv = qkv + 3 * elements; - - // When there is no global token, no need to process global Q, K and V - if (max_num_global > 0 && nullptr != global_input) { - if (!LaunchTransQkv(stream, - 3, - sequence_length, - batch_size, - head_size, - num_heads, - max_threads_per_block, - false, - global_input, - global_qkv)) { - return false; + if (max_num_global > 0 && nullptr != global_input) { + LaunchAddBiasTranspose(stream, 3, format, max_threads_per_block, batch_size, + sequence_length, num_heads, head_size, + global_input, global_bias, qkv + 3 * elements, + use_half4); } - } + } else { + LaunchAddBiasTranspose(stream, 5, format, max_threads_per_block, batch_size, + sequence_length, num_heads, head_size, + input, bias, qkv, + use_half4); - // Now qkv has Q, K, V: each has size BxNxSxH + compact_global_q = (disable_compact_memory == false); + LaunchAddBiasTranspose(stream, 1, format, max_threads_per_block, batch_size, + compact_global_q ? max_num_global : sequence_length, num_heads, head_size, + global_input + 2 * elements, global_bias, qkv + 5 * elements, + use_half4); + } + CUDA_RETURN_IF_ERROR(cudaGetLastError()); + + // Transposed Q, K, V with shape (B, N, S, H) const T* q = qkv; const T* k = q + elements; const T* v = k + elements; - const T* global_q = global_qkv; - const T* global_k = global_q + elements; - const T* global_v = global_k + elements; + // Transposed global Q, K, V with shape (B, N, S, H). + // When compact_global_q is true, Global Q has actual shape (B, N, G, H) although we allocated space of (B, N, S, H) + // When max_num_global == 0, these pointers are not used in GEMM so the value does not matter. + const T* global_q = (format == 1 ? v + elements : qkv + 5 * elements); + const T* global_k = (format == 1 ? global_q + elements : qkv + 3 * elements); + const T* global_v = (format == 1 ? global_k + elements : qkv + 4 * elements); // Q*K' are scaled by 1/sqrt(H) const float rsqrt_head_size = 1.f / sqrt(static_cast(head_size)); @@ -905,58 +912,56 @@ bool LongformerQkvToContext( T* temp_output = qkv; // Q will be overwritten if (disable_compact_memory) { - if (!LaunchLongformerSoftmaxSimpleKernel( + ORT_RETURN_IF_ERROR(LaunchLongformerSoftmaxSimpleKernel( stream, cublas, - workspace, // softmax space - q, // transposed Q with shape (B, N, S, H) - k, // transposed K with shape (B, N, S, H) - v, // transposed V with shape (B, N, S, H) - attention_mask, // attention mask with shape (B, S), with value 0.0 not masked, and -10000.0 masked. - global_q, // Q for global tokens with shape (B, N, S, H) - global_k, // K for global tokens with shape (B, N, S, H) - global_v, // V for global tokens with shape (B, N, S, H) - global_attention, // global attention flags with shape (B, S), with value 0 for local and 1 for global. - global_index, // Global index with shape (B, S) - batch_global_num, // Number of global tokens per batch with shape (B, 1) - pinned_buffer, // Pinned memory in CPU. Number of global tokens per batch with shape (B, 1) - temp_output, // output with shape (B, N, S, H) - rsqrt_head_size, // scalar - batch_size, // batch size - sequence_length, // sequence length - num_heads, // number of heads - head_size, // hidden size per head - window, // Half (one-sided) window size - element_size)) { - return false; - } + workspace, + q, + k, + v, + attention_mask, + global_q, + global_k, + global_v, + global_attention, + global_index, + batch_global_num, + pinned_buffer, + temp_output, + rsqrt_head_size, + batch_size, + sequence_length, + num_heads, + head_size, + window, + element_size)); } else { - assert(max_num_global <= window); - if (!LaunchLongformerSoftmaxKernel( + ORT_ENFORCE(max_num_global <= window); + ORT_RETURN_IF_ERROR(LaunchLongformerSoftmaxKernel( stream, cublas, - workspace, // softmax space - q, // Transposed Q with shape B x N x S x H - k, // Transposed K with shape B x N x S x H - v, // Transposed V with shape B x N x S x H - attention_mask, // Attention mask flags with shape B x S. Value -10000.0 means masked, and 0.0 not mased. - global_q, // Transposed global Q with shape B x N x S x H. - global_k, // Transposed global K with shape B x N x S x H - global_v, // Transposed global V with shape B x N x S x H - global_attention, // Global attention flags with shape B x S - global_index, // Global index with shape B x S - batch_global_num, // Number of global token per batch with shape B x 1 - pinned_buffer, // Pinned Memory Buffer - temp_output, // Output with shape B x N x S x H - rsqrt_head_size, // Scaler - batch_size, // Batch size - sequence_length, // Sequence length - num_heads, // Number of attention heads - head_size, // Hidden size per head - window, // Half (one-sided) window size - element_size)) { - return false; - } + workspace, + q, + k, + v, + attention_mask, + max_num_global, + compact_global_q, + global_q, + global_k, + global_v, + global_attention, + global_index, + batch_global_num, + pinned_buffer, + temp_output, + rsqrt_head_size, + batch_size, + sequence_length, + num_heads, + head_size, + window, + element_size)); } // The temp_output is BxNxSxH, transpose it to final output BxSxNxH @@ -964,13 +969,15 @@ bool LongformerQkvToContext( num_heads, max_threads_per_block, false, temp_output, output); } -bool LaunchLongformerAttentionKernel( +Status LaunchLongformerAttentionKernel( const cudaDeviceProp& device_prop, cublasHandle_t cublas, cudaStream_t stream, const void* input, + const void* bias, const void* attention_mask, const void* global_input, + const void* global_bias, const int* global_attention, const int* global_index, const int* batch_global_num, @@ -984,7 +991,9 @@ bool LaunchLongformerAttentionKernel( int window, int max_num_global, const size_t element_size, - bool disable_compact_memory) { + bool disable_compact_memory, + bool use_merged_qkv_weights, + bool use_half4) { CublasMathModeSetter helper(device_prop, cublas, CUBLAS_TENSOR_OP_MATH); size_t softmax_workspace_size = GetLongformerSoftmaxWorkspaceSize(element_size, batch_size, @@ -996,8 +1005,10 @@ bool LaunchLongformerAttentionKernel( return LongformerQkvToContext(device_prop, cublas, stream, batch_size, sequence_length, num_heads, head_size, window, element_size, reinterpret_cast(input), + reinterpret_cast(bias), reinterpret_cast(attention_mask), reinterpret_cast(global_input), + reinterpret_cast(global_bias), global_attention, global_index, batch_global_num, @@ -1006,13 +1017,17 @@ bool LaunchLongformerAttentionKernel( reinterpret_cast(workspace), reinterpret_cast(output), softmax_workspace_size, - disable_compact_memory); + disable_compact_memory, + use_merged_qkv_weights, + use_half4); } else { return LongformerQkvToContext(device_prop, cublas, stream, batch_size, sequence_length, num_heads, head_size, window, element_size, reinterpret_cast(input), + reinterpret_cast(bias), reinterpret_cast(attention_mask), reinterpret_cast(global_input), + reinterpret_cast(global_bias), global_attention, global_index, batch_global_num, @@ -1021,7 +1036,9 @@ bool LaunchLongformerAttentionKernel( reinterpret_cast(workspace), reinterpret_cast(output), softmax_workspace_size, - disable_compact_memory); + disable_compact_memory, + use_merged_qkv_weights, + false); } } diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.h b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.h index 7aa89e3e13..c9143e07a2 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.h +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_impl.h @@ -9,25 +9,28 @@ namespace contrib { namespace cuda { size_t GetPinnedBufferSize( - int batch_size); + size_t batch_size); size_t GetLongformerAttentionWorkspaceSize( size_t element_size, - int batch_size, - int num_heads, - int head_size, - int sequence_length, - int max_num_global, - int window, + size_t batch_size, + size_t num_heads, + size_t head_size, + size_t sequence_length, + + size_t max_num_global, + size_t window, bool disable_compact_memory); -bool LaunchLongformerAttentionKernel( +Status LaunchLongformerAttentionKernel( const cudaDeviceProp& device_prop, // Device Properties cublasHandle_t cublas, // Cublas handle cudaStream_t stream, // CUDA stream const void* input, // Input tensor + const void* bias, // Bias tensor const void* attention_mask, // Attention mask with shape (B, S) const void* global_input, // Global attention input, or nullptr when max_num_global == 0. + const void* global_bias, // Global bias tensor const int* global_attention, // Global attention flags with shape (B, S) const int* global_index, // Global index const int* batch_global_num, // Number of global tokens per batch. It is in device memory. @@ -41,8 +44,9 @@ bool LaunchLongformerAttentionKernel( int window, // One sided attention window (W) int max_num_global, // Maximum number of global tokens (G) const size_t element_size, // Element size of input tensor, - bool disable_compact_memory // Disable compact memory kernel -); + bool disable_compact_memory, // Disable compact memory kernel + bool use_merged_qkv_weights, + bool use_half4); } // namespace cuda } // namespace contrib diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.cu b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.cu index 831d4867b1..21de67e52d 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.cu +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.cu @@ -31,10 +31,7 @@ limitations under the License. using namespace onnxruntime::cuda; using namespace cub; -#define CHECK(expr) \ - if (!CUBLAS_CALL(expr)) { \ - return false; \ - } +#define CHECK(expr) CUBLAS_RETURN_IF_ERROR(expr) namespace onnxruntime { namespace contrib { @@ -191,7 +188,7 @@ __launch_bounds__(blockSize) } // Launch the softmax kernel for non compact memory. -bool LaunchLongformerSoftmaxSimpleKernel( +Status LaunchLongformerSoftmaxSimpleKernel( cudaStream_t stream, cublasHandle_t cublas, void* workspace, // softmax space @@ -270,8 +267,9 @@ bool LaunchLongformerSoftmaxSimpleKernel( // The results are stored in scratch1. int w = attention_window; - int x_offset = num_heads * sequence_length * head_size; - int y_offset = num_heads * sequence_length * sequence_length; + size_t x_offset = static_cast(num_heads) * sequence_length * head_size; + // Use size_t to avoid integer overflow since B x N x S x S is 12G for B=64, N=12, S=4096 + size_t y_offset = static_cast(num_heads) * sequence_length * sequence_length; int last_block = (sequence_length / w) - 1; int strideA = sequence_length * head_size; int strideB = sequence_length * head_size; @@ -308,10 +306,11 @@ bool LaunchLongformerSoftmaxSimpleKernel( } else { // sequence_length > 2 * w for (int i = 0; i < batch_size; ++i) { for (int j = 0; j < num_heads; ++j) { - const void* q_head = reinterpret_cast(q) + \ + const void* q_head = reinterpret_cast(q) + (i * x_offset + j * sequence_length * head_size + w * head_size) * element_size; - const void* k_head = reinterpret_cast(k) + (i * x_offset + j * sequence_length * head_size) * element_size; - void* qk_head = reinterpret_cast(scratch1) + \ + const void* k_head = reinterpret_cast(k) + + (i * x_offset + j * sequence_length * head_size) * element_size; + void* qk_head = reinterpret_cast(scratch1) + (i * y_offset + j * sequence_length * sequence_length + w * sequence_length) * element_size; int count = (sequence_length - 2 * w) / w; CHECK(cublasGemmStridedBatchedEx(cublas, @@ -366,7 +365,7 @@ bool LaunchLongformerSoftmaxSimpleKernel( const void* q_head = reinterpret_cast(q) + (last_block * w * head_size) * element_size; const void* k_head = reinterpret_cast(k) + ((last_block - 1) * w * head_size) * element_size; - void* qk_head = reinterpret_cast(scratch1) + \ + void* qk_head = reinterpret_cast(scratch1) + (last_block * w * sequence_length + (last_block - 1) * w) * element_size; CHECK(cublasGemmStridedBatchedEx(cublas, CUBLAS_OP_T, @@ -425,7 +424,7 @@ bool LaunchLongformerSoftmaxSimpleKernel( resultType, algo)); - const void* global_q_batch = reinterpret_cast(global_q) + \ + const void* global_q_batch = reinterpret_cast(global_q) + (i * num_heads * sequence_length * head_size) * element_size; const void* global_k_batch = reinterpret_cast(global_k) + (i * x_offset) * element_size; int strideB_global = sequence_length * head_size; @@ -516,11 +515,11 @@ bool LaunchLongformerSoftmaxSimpleKernel( } else { // sequence_length > 2 * w for (int i = 0; i < batch_size; ++i) { for (int j = 0; j < num_heads; ++j) { - const void* v_head = reinterpret_cast(v) + \ + const void* v_head = reinterpret_cast(v) + (i * x_offset + j * head_size * sequence_length) * element_size; - const void* prob_head = reinterpret_cast(softmax_out) + \ - (i * y_offset + j * sequence_length * sequence_length + w * sequence_length) * element_size; - void* out_head = reinterpret_cast(output) + \ + size_t offset = (i * y_offset + j * sequence_length * sequence_length + w * sequence_length) * element_size; + const void* prob_head = reinterpret_cast(softmax_out) + offset; + void* out_head = reinterpret_cast(output) + (i * x_offset + j * head_size * sequence_length + w * head_size) * element_size; int count = (sequence_length - 2 * w) / w; CHECK(cublasGemmStridedBatchedEx(cublas, @@ -574,7 +573,7 @@ bool LaunchLongformerSoftmaxSimpleKernel( algo)); const void* v_head = reinterpret_cast(v) + (last_block - 1) * w * head_size * element_size; - const void* prob_head = reinterpret_cast(softmax_out) + \ + const void* prob_head = reinterpret_cast(softmax_out) + (sequence_length * last_block * w + (last_block - 1) * w) * element_size; void* out_head = reinterpret_cast(output) + last_block * w * head_size * element_size; @@ -608,7 +607,7 @@ bool LaunchLongformerSoftmaxSimpleKernel( int glob_longdim_mm = (last_block - 1) * w; const void* v_head = reinterpret_cast(v) + (i * x_offset) * element_size; - const void* prob_head = reinterpret_cast(softmax_out) + \ + const void* prob_head = reinterpret_cast(softmax_out) + (i * y_offset + 2 * w * sequence_length) * element_size; void* out_head = reinterpret_cast(output) + (i * x_offset + 2 * w * head_size) * element_size; @@ -667,7 +666,7 @@ bool LaunchLongformerSoftmaxSimpleKernel( } } - return true; + return Status::OK(); } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.h b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.h index 341935aad5..cbf86e6601 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.h +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_attention_softmax.h @@ -13,13 +13,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#pragma once +#include "core/common/common.h" namespace onnxruntime { namespace contrib { namespace cuda { // Launch the softmax kernels that does not use compact memory. -bool LaunchLongformerSoftmaxSimpleKernel( +Status LaunchLongformerSoftmaxSimpleKernel( cudaStream_t stream, cublasHandle_t cublas, void* workspace, // softmax space diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.cu b/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.cu index 492c24330f..f25c8dfc98 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.cu @@ -17,6 +17,7 @@ limitations under the License. #include #include #include "core/providers/cuda/cuda_common.h" +#include "core/providers/cuda/cu_inc/common.cuh" #include "longformer_global_impl.h" using namespace onnxruntime::cuda; @@ -26,20 +27,20 @@ namespace onnxruntime { namespace contrib { namespace cuda { -size_t GetGlobalScratchSize(int batch_size, int sequence_length) { +size_t GetGlobalScratchSize(int sequence_length) { // Global Index scratch layout: - // [sequence_index: int BxS][tmp_storage: int 1024x1] - return sizeof(int) * (batch_size * sequence_length + 1024); + // [sequence_index: int S][tmp_storage: int 1024x1] + return sizeof(int) * (sequence_length + 1024); } __global__ void InitSequenceIndexKernel(int* sequence_index, int sequence_length) { - int batch_index = blockIdx.x; - for (int i = threadIdx.x; i < sequence_length; i += blockDim.x) { - sequence_index[batch_index * sequence_length + i] = i; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < sequence_length; i += blockDim.x) { + sequence_index[i] = i; } } -void BuildGlobalIndex( +Status BuildGlobalIndex( + const cudaDeviceProp& device_prop, cudaStream_t stream, const int* global_attention, int batch_size, @@ -49,27 +50,31 @@ void BuildGlobalIndex( void* scratch, size_t scratch_size) { int* sequence_index = (int*)scratch; - int* tmp_storage = sequence_index + batch_size * sequence_length; + int* tmp_storage = sequence_index + sequence_length; - InitSequenceIndexKernel<<>>(sequence_index, sequence_length); + const int threads = device_prop.maxThreadsPerBlock; + int blocks = CeilDiv(sequence_length, threads); + InitSequenceIndexKernel<<>>(sequence_index, sequence_length); // Determine temporary device storage size. - // For int* inputs/outputs, it need 767 bytes. When data type changes, its size will be different. + // For int* inputs/outputs, it need 767 bytes. We reserved 1024*4 bytes, which shall be enough. size_t temp_storage_bytes = 0; - cub::DevicePartition::Flagged(NULL, temp_storage_bytes, sequence_index, - global_attention, global_index, batch_global_num, sequence_length, stream); - if (temp_storage_bytes + sizeof(int) * batch_size * sequence_length > scratch_size) { + CUDA_RETURN_IF_ERROR(cub::DevicePartition::Flagged( + NULL, temp_storage_bytes, sequence_index, + global_attention, global_index, batch_global_num, sequence_length, stream)); + if (temp_storage_bytes + sizeof(int) * sequence_length > scratch_size) { ORT_THROW("LongformerAttention scratch space is not large enough. Temp storage bytes are", temp_storage_bytes); } // Find the global attention indices and number of global attention tokens for (int i = 0; i < batch_size; ++i) { - cub::DevicePartition::Flagged(reinterpret_cast(tmp_storage), temp_storage_bytes, sequence_index, - global_attention + i * sequence_length, global_index + i * sequence_length, - batch_global_num + i, sequence_length, stream); + CUDA_RETURN_IF_ERROR(cub::DevicePartition::Flagged( + reinterpret_cast(tmp_storage), temp_storage_bytes, sequence_index, + global_attention + i * sequence_length, global_index + i * sequence_length, + batch_global_num + i, sequence_length, stream)); } - return; + return Status::OK(); } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.h b/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.h index 4e3bd13cd4..a8d6988ec7 100644 --- a/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.h +++ b/onnxruntime/contrib_ops/cuda/bert/longformer_global_impl.h @@ -8,10 +8,11 @@ namespace contrib { namespace cuda { // Size of global Index scratch in bytes. -size_t GetGlobalScratchSize(int batch_size, int sequence_length); +size_t GetGlobalScratchSize(int sequence_length); // Find the global attention indices and number of global attention tokens -void BuildGlobalIndex( +Status BuildGlobalIndex( + const cudaDeviceProp& device_prop, cudaStream_t stream, const int* global_attention, int batch_size, diff --git a/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.cc b/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.cc index 53fb3ffa9d..b9a8613c85 100644 --- a/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.cc +++ b/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.cc @@ -66,6 +66,6 @@ Status NGramRepeatBlock::ComputeInternal(OpKernelContext* context) const { return Status::OK(); } -} //namespace cuda +} // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.h b/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.h index eb219916ba..139447d37b 100644 --- a/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.h +++ b/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block.h @@ -15,6 +15,7 @@ class NGramRepeatBlock final : public CudaKernel { public: NGramRepeatBlock(const OpKernelInfo& op_kernel_info); Status ComputeInternal(OpKernelContext* ctx) const override; + private: int64_t ngram_size_; }; diff --git a/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block_impl.cu b/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block_impl.cu index 887f32721c..8a04ede231 100644 --- a/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/ngram_repeat_block_impl.cu @@ -33,8 +33,8 @@ __global__ void banRepeatedTokens(const int64_t* __restrict__ tokens, extern __shared__ int64_t tokens_shm[]; tokens_shm[col] = tokens[start]; if (col == blockDim.x - 1) { - for (int i=1; i::ComputeInternal(OpKernelContext* ctx) const { size_t element_size = sizeof(T); typedef typename ToCudaType::MappedType CudaT; - if (!LaunchSkipLayerNormKernel( + return LaunchSkipLayerNormKernel( Stream(), reinterpret_cast(output->MutableData()), reinterpret_cast(input->Data()), @@ -107,15 +107,9 @@ Status SkipLayerNorm::ComputeInternal(OpKernelContext* ctx) const { epsilon_, hidden_size, static_cast(element_count), - element_size)) { - // Get last error to reset it to cudaSuccess. - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + element_size); } -} //namespace cuda +} // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.cu b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.cu index 49bebfec07..adbdcc9249 100644 --- a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.cu @@ -35,15 +35,15 @@ namespace onnxruntime { namespace contrib { namespace cuda { -template +template T maybe2half(float x); -template<> +template <> float maybe2half(float x) { return x; } -template<> +template <> half maybe2half(float x) { return __float2half_rn(x); } @@ -98,9 +98,9 @@ __global__ void SkipLayerNormKernelSmall( if (ILP * threadIdx.x < ld) { T rldval_sum = T(0.f); T rldvalsq_sum = T(0.f); - #pragma unroll +#pragma unroll for (int i = 0; i < ILP; i++) { - input_v[i] += hasBias ? skip_v[i] + bias_v[i]: skip_v[i]; + input_v[i] += hasBias ? skip_v[i] + bias_v[i] : skip_v[i]; const T rldval = rld * input_v[i]; rldval_sum += rldval; rldvalsq_sum += rldval * input_v[i]; @@ -111,11 +111,10 @@ __global__ void SkipLayerNormKernelSmall( } template -bool LaunchSkipLayerNormKernel( +Status LaunchSkipLayerNormKernel( cudaStream_t stream, T* output, const T* input, const T* skip, const T* gamma, const T* beta, const T* bias, float epsilon, const int ld, const int element_count, size_t element_size) { - // this must be true because n is the total size of the tensor assert(element_count % ld == 0); bool hasBias = (bias == nullptr) ? false : true; @@ -125,37 +124,37 @@ bool LaunchSkipLayerNormKernel( constexpr int block_size = 32; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 64) { constexpr int block_size = 64 / 2; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 128) { constexpr int block_size = 128 / 4; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 384) { constexpr int block_size = 384 / 4; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 768) { constexpr int block_size = 768 / 4; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 1024) { constexpr int block_size = 1024 / 4; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else { constexpr int block_size = 256; SkipLayerNormKernel <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output); + maybe2half(epsilon), output); } } else { const int grid_size = element_count / ld; @@ -163,41 +162,41 @@ bool LaunchSkipLayerNormKernel( constexpr int block_size = 32; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 64) { constexpr int block_size = 64; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld <= 128) { constexpr int block_size = 128; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else if (ld == 384) { constexpr int block_size = 384; SkipLayerNormKernelSmall <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output, hasBias); + maybe2half(epsilon), output, hasBias); } else { constexpr int block_size = 256; SkipLayerNormKernel <<>>(ld, input, skip, beta, gamma, bias, - maybe2half(epsilon), output); + maybe2half(epsilon), output); } } - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } -template bool LaunchSkipLayerNormKernel(cudaStream_t stream, float* output, const float* input, +template Status LaunchSkipLayerNormKernel(cudaStream_t stream, float* output, const float* input, const float* skip, const float* gamma, const float* beta, const float* bias, float epsilon, const int ld, const int element_count, size_t element_size); -template bool LaunchSkipLayerNormKernel(cudaStream_t stream, half* output, const half* input, - const half* skip, const half* gamma, const half* beta, - const half* bias, float epsilon, const int ld, - const int element_count, size_t element_size); +template Status LaunchSkipLayerNormKernel(cudaStream_t stream, half* output, const half* input, + const half* skip, const half* gamma, const half* beta, + const half* bias, float epsilon, const int ld, + const int element_count, size_t element_size); } // namespace cuda } // namespace contrib diff --git a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.h b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.h index ace02b63fd..b381eaa308 100644 --- a/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.h +++ b/onnxruntime/contrib_ops/cuda/bert/skip_layer_norm_impl.h @@ -2,27 +2,26 @@ // Licensed under the MIT License. #pragma once +#include "core/common/common.h" namespace onnxruntime { namespace contrib { namespace cuda { template -bool LaunchSkipLayerNormKernel( +Status LaunchSkipLayerNormKernel( cudaStream_t stream, - T* output, // output tensor - const T* input, // input tensor - const T* skip, // skip tensor - const T* gamma, // Layer normalization gamma tensor - const T* beta, // Layer normalization beta tensor - const T* bias, // Layer normalization beta tensor + T* output, // output tensor + const T* input, // input tensor + const T* skip, // skip tensor + const T* gamma, // Layer normalization gamma tensor + const T* beta, // Layer normalization beta tensor + const T* bias, // Layer normalization beta tensor float epsilon, // Layer normalization epsilon - int hidden_size, // hidden size, it is the leading dimension (ld) - int element_count, // number of elements in input tensor - size_t element_size -); + int hidden_size, // hidden size, it is the leading dimension (ld) + int element_count, // number of elements in input tensor + size_t element_size); } // namespace cuda } // namespace contrib } // namespace onnxruntime - diff --git a/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc b/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc index 847ad11cbf..0c22bc5a29 100644 --- a/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc +++ b/onnxruntime/contrib_ops/cuda/bert/transformer_common.cc @@ -34,4 +34,4 @@ const TransformerOptions* TransformerOptions::GetInstance() { } // namespace cuda } // namespace contrib -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/transformer_common.h b/onnxruntime/contrib_ops/cuda/bert/transformer_common.h index c2372137f6..f0e46932d3 100644 --- a/onnxruntime/contrib_ops/cuda/bert/transformer_common.h +++ b/onnxruntime/contrib_ops/cuda/bert/transformer_common.h @@ -43,4 +43,4 @@ class TransformerOptions { } // namespace cuda } // namespace contrib -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc index 7062398dda..d9e9e6dc55 100644 --- a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc @@ -97,6 +97,9 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, TransposeMatMul); // backward compatibility class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BFloat16, FusedMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kOnnxDomain, 1, BFloat16_float_BFloat16, LayerNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, QOrderedMatMul); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, QOrderedLayerNormalization); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, QOrderedGelu); #ifdef ENABLE_ATEN class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kPytorchAtenDomain, 1, ATen); @@ -201,6 +204,9 @@ Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, #ifdef ENABLE_ATEN BuildKernelCreateInfo, diff --git a/onnxruntime/contrib_ops/cuda/fused_conv.cc b/onnxruntime/contrib_ops/cuda/fused_conv.cc index 975a73d212..fa3b33166f 100644 --- a/onnxruntime/contrib_ops/cuda/fused_conv.cc +++ b/onnxruntime/contrib_ops/cuda/fused_conv.cc @@ -37,7 +37,7 @@ class FusedConv : public onnxruntime::cuda::Conv { Status ComputeInternal(OpKernelContext* context) const override { CUDNN_RETURN_IF_ERROR(status_); std::lock_guard lock(Base::s_.mutex); - ORT_RETURN_IF_ERROR(Base::UpdateState(context, true)); + ORT_RETURN_IF_ERROR(Base::UpdateState(context)); if (Base::s_.Y->Shape().Size() == 0) { return Status::OK(); } @@ -47,25 +47,27 @@ class FusedConv : public onnxruntime::cuda::Conv { const auto alpha = onnxruntime::cuda::Consts::One; const auto beta = onnxruntime::cuda::Consts::Zero; IAllocatorUniquePtr workspace = Base::GetWorkSpace(); - auto cudnn_status = cudnnConvolutionBiasActivationForward(Base::CudnnHandle(), - &alpha, - Base::s_.x_tensor, - Base::s_.x_data, - Base::s_.w_desc, - Base::s_.w_data, - Base::s_.conv_desc, - Base::s_.algo, - workspace.get(), - Base::s_.workspace_bytes, - has_z ? &alpha : &beta, - has_z ? Base::s_.z_tensor : Base::s_.y_tensor, - has_z ? Base::s_.z_data : Base::s_.y_data, - Base::s_.b_tensor, - has_b ? Base::s_.b_data : Base::s_.b_zero, - activation_desc_, - Base::s_.y_tensor, - Base::s_.y_data); - if (CUDNN_STATUS_SUCCESS != cudnn_status) { + + if (has_b && has_z && !Base::s_.post_slicing_required) { + CUDNN_RETURN_IF_ERROR(cudnnConvolutionBiasActivationForward(Base::CudnnHandle(), + &alpha, + Base::s_.x_tensor, + Base::s_.x_data, + Base::s_.w_desc, + Base::s_.w_data, + Base::s_.conv_desc, + Base::s_.algo, + workspace.get(), + Base::s_.workspace_bytes, + &alpha, + Base::s_.z_tensor, + Base::s_.z_data, + Base::s_.b_tensor, + Base::s_.b_data, + activation_desc_, + Base::s_.y_tensor, + Base::s_.y_data)); + } else { CUDNN_RETURN_IF_ERROR(cudnnConvolutionForward(Base::CudnnHandle(), &alpha, Base::s_.x_tensor, @@ -79,21 +81,38 @@ class FusedConv : public onnxruntime::cuda::Conv { &beta, Base::s_.y_tensor, Base::s_.y_data)); - if (has_b) { - CUDNN_RETURN_IF_ERROR(cudnnAddTensor(Base::CudnnHandle(), &alpha, Base::s_.b_tensor, Base::s_.b_data, - &alpha, Base::s_.y_tensor, Base::s_.y_data)); + + if (Base::s_.post_slicing_required) { + ORT_RETURN_IF_ERROR(onnxruntime::cuda::SliceOutUnwantedOutputSection( + this->Stream(), Base::s_.y_data, Base::s_.y_dims_with_adjusted_pads, Base::s_.Y->MutableDataRaw(), + Base::s_.y_dims.GetDims(), Base::s_.slice_starts, Base::s_.slice_ends, Base::s_.slice_axes, Base::s_.element_size)); + + onnxruntime::cuda::CudnnTensor sliced_y_tensor; + ORT_RETURN_IF_ERROR(sliced_y_tensor.Set(Base::s_.y_dims.GetDims(), onnxruntime::cuda::CudnnTensor::GetDataType())); + + if (has_b) { + CUDNN_RETURN_IF_ERROR(cudnnAddTensor(Base::CudnnHandle(), &alpha, Base::s_.b_tensor, Base::s_.b_data, + &alpha, sliced_y_tensor, Base::s_.Y->MutableDataRaw())); + } + if (has_z) { + CUDNN_RETURN_IF_ERROR(cudnnAddTensor(Base::CudnnHandle(), &alpha, Base::s_.z_tensor, Base::s_.z_data, + &alpha, sliced_y_tensor, Base::s_.Y->MutableDataRaw())); + } + + CUDNN_RETURN_IF_ERROR(cudnnActivationForward(Base::CudnnHandle(), activation_desc_, &alpha, sliced_y_tensor, + Base::s_.y_data, &beta, sliced_y_tensor, Base::s_.y_data)); + } else { + if (has_b) { + CUDNN_RETURN_IF_ERROR(cudnnAddTensor(Base::CudnnHandle(), &alpha, Base::s_.b_tensor, Base::s_.b_data, + &alpha, Base::s_.y_tensor, Base::s_.y_data)); + } + if (has_z) { + CUDNN_RETURN_IF_ERROR(cudnnAddTensor(Base::CudnnHandle(), &alpha, Base::s_.z_tensor, Base::s_.z_data, + &alpha, Base::s_.y_tensor, Base::s_.y_data)); + } + CUDNN_RETURN_IF_ERROR(cudnnActivationForward(Base::CudnnHandle(), activation_desc_, &alpha, Base::s_.y_tensor, + Base::s_.y_data, &beta, Base::s_.y_tensor, Base::s_.y_data)); } - if (has_z) { - CUDNN_RETURN_IF_ERROR(cudnnAddTensor(Base::CudnnHandle(), &alpha, Base::s_.z_tensor, Base::s_.z_data, - &alpha, Base::s_.y_tensor, Base::s_.y_data)); - } - CUDNN_RETURN_IF_ERROR(cudnnActivationForward(Base::CudnnHandle(), activation_desc_, &alpha, Base::s_.y_tensor, - Base::s_.y_data, &beta, Base::s_.y_tensor, Base::s_.y_data)); - } - if (Base::s_.post_slicing_required) { - ORT_RETURN_IF_ERROR(onnxruntime::cuda::SliceOutUnwantedOutputSection( - this->Stream(), Base::s_.y_data, Base::s_.y_dims_with_adjusted_pads, Base::s_.Y->MutableDataRaw(), - Base::s_.y_dims.GetDims(), Base::s_.slice_starts, Base::s_.slice_ends, Base::s_.slice_axes, Base::s_.element_size)); } return Status::OK(); } diff --git a/onnxruntime/contrib_ops/cuda/layer_norm.cc b/onnxruntime/contrib_ops/cuda/layer_norm.cc index f1b70a295a..6a27683001 100644 --- a/onnxruntime/contrib_ops/cuda/layer_norm.cc +++ b/onnxruntime/contrib_ops/cuda/layer_norm.cc @@ -43,7 +43,7 @@ Status LayerNorm::ComputeInternal(OpKernelContext* ctx) con typedef typename ToCudaType::MappedType CudaT; typedef typename ToCudaType::MappedType CudaU; typedef typename ToCudaType::MappedType CudaV; - //Inputs + // Inputs const Tensor* X = ctx->Input(0); const Tensor* scale = ctx->Input(1); const Tensor* bias = ctx->Input(2); @@ -64,7 +64,7 @@ Status LayerNorm::ComputeInternal(OpKernelContext* ctx) con Tensor* Y = ctx->Output(0, x_shape); auto Y_data = reinterpret_cast(Y->MutableData()); - //Mean and variance + // Mean and variance std::vector mean_inv_std_var_dim; for (int i = 0; i < static_cast(x_shape.NumDimensions()); ++i) { if (i < axis) { @@ -98,6 +98,6 @@ Status LayerNorm::ComputeInternal(OpKernelContext* ctx) con return Status::OK(); } -} //namespace cuda +} // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/layer_norm_impl.cu b/onnxruntime/contrib_ops/cuda/layer_norm_impl.cu index 748f519539..b1ebb0b3a5 100644 --- a/onnxruntime/contrib_ops/cuda/layer_norm_impl.cu +++ b/onnxruntime/contrib_ops/cuda/layer_norm_impl.cu @@ -114,8 +114,8 @@ __device__ void cuWelfordMuSigma2( U curr = static_cast(lvals[l]); cuWelfordOnlineSum(curr, mu, sigma2, count); } - // intra-warp reductions - #pragma unroll +// intra-warp reductions +#pragma unroll for (int stride = GPU_WARP_SIZE / 2; stride > 0; stride /= 2) { U muB = WARP_SHFL_DOWN(mu, stride); U countB = WARP_SHFL_DOWN(count, stride); @@ -209,8 +209,8 @@ __device__ void cuWelfordMuSigma2( float curr = static_cast(lvals[l]); cuWelfordOnlineSum(curr, mu, sigma2, count); } - // intra-warp reductions - #pragma unroll +// intra-warp reductions +#pragma unroll for (int stride = GPU_WARP_SIZE / 2; stride > 0; stride /= 2) { float muB = WARP_SHFL_DOWN(mu, stride); float countB = WARP_SHFL_DOWN(count, stride); diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc b/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc index 3dbb2222da..dd3e58e547 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc @@ -19,14 +19,13 @@ namespace { template struct DispatchBiasSoftmaxImpl { Status operator()(cudaStream_t stream, cudnnHandle_t cudnn_handle, Tensor* Y, const Tensor* X, const Tensor* B, - int element_count, int batch_count, bool is_inner_broadcast, int bias_broadcast_size, - size_t element_size) { + int element_count, int batch_count, bool is_inner_broadcast, int bias_broadcast_size) { typedef typename ToCudaType::MappedType CudaT; CudaT* output_data = reinterpret_cast(Y->template MutableData()); const CudaT* input_data = reinterpret_cast(X->template Data()); const CudaT* bias_data = reinterpret_cast(B->template Data()); return BiasSoftmaxImpl(stream, cudnn_handle, output_data, input_data, bias_data, element_count, batch_count, - is_inner_broadcast, bias_broadcast_size, element_size); + is_inner_broadcast, bias_broadcast_size); } }; @@ -34,16 +33,14 @@ struct DispatchBiasSoftmaxImpl { // MIOpen doesn't support double so ROCm kernel doesn't have double support for now. #ifdef USE_ROCM -#define BIAS_SOFTMAX_KERNEL_DATATYPES \ - { DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType() } -#define BIAS_SOFTMAX_DISPATCH_TYPES float, MLFloat16 +#define BIAS_SOFTMAX_TYPES float, MLFloat16 #else -#define BIAS_SOFTMAX_KERNEL_DATATYPES DataTypeImpl::AllIEEEFloatTensorTypes() -#define BIAS_SOFTMAX_DISPATCH_TYPES float, MLFloat16, double +#define BIAS_SOFTMAX_TYPES float, MLFloat16, double #endif -ONNX_OPERATOR_KERNEL_EX(BiasSoftmax, kMSDomain, 1, kCudaExecutionProvider, - (*KernelDefBuilder::Create()).TypeConstraint("T", BIAS_SOFTMAX_KERNEL_DATATYPES), BiasSoftmax); +ONNX_OPERATOR_KERNEL_EX( + BiasSoftmax, kMSDomain, 1, kCudaExecutionProvider, + (*KernelDefBuilder::Create()).TypeConstraint("T", BuildKernelDefConstraints()), BiasSoftmax); Status BiasSoftmax::ComputeInternal(OpKernelContext* ctx) const { const Tensor* X = ctx->Input(0); @@ -57,12 +54,13 @@ Status BiasSoftmax::ComputeInternal(OpKernelContext* ctx) const { const int element_count = static_cast(X_shape.SizeFromDimension(axis)); int bias_broadcast_size = static_cast(B_shape.Size() / element_count); if (is_inner_broadcast_) bias_broadcast_size = batch_count / bias_broadcast_size; - utils::MLTypeCallDispatcher t_disp(X->GetElementType()); + utils::MLTypeCallDispatcher t_disp(X->GetElementType()); return t_disp.InvokeRet(Stream(), CudnnHandle(), Y, X, B, element_count, batch_count, - is_inner_broadcast_, bias_broadcast_size, - X->DataType()->Size()); + is_inner_broadcast_, bias_broadcast_size); } +#undef BIAS_SOFTMAX_TYPES + } // namespace cuda } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu index b93f2f7722..427c7fc624 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu @@ -10,9 +10,9 @@ #include "core/providers/cuda/cu_inc/binary_elementwise_impl.cuh" #include "core/providers/cuda/cu_inc/common.cuh" #include "core/providers/cuda/math/binary_elementwise_ops_impl_functors.cuh" +#include "core/providers/cuda/math/softmax_common.h" #include "core/providers/cuda/math/softmax_warpwise_impl.cuh" #include "core/providers/cuda/shared_inc/accumulation_type.h" -#include "core/providers/cuda/shared_inc/softmax.h" using namespace onnxruntime; using namespace onnxruntime::cuda; @@ -126,9 +126,9 @@ __global__ void BiasSoftmaxWarpForward(output_t* output, const input_t* input, c template Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* output_data, const T* input_data, const T* bias_data, int element_count, int batch_count, bool is_inner_broadcast, - int bias_broadcast_size, size_t element_size) { + int bias_broadcast_size) { if (element_count == 0) return Status::OK(); - if (element_count <= 1024 && element_count * static_cast(element_size) <= 4096) { + if (element_count <= 1024 && element_count * static_cast(sizeof(T)) <= 4096) { typedef AccumulationType_t AccT; int log2_elements = log2_ceil(element_count); const int next_power_of_two = 1 << log2_elements; @@ -215,7 +215,7 @@ Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* outpu static_cast(batch_count * element_count)); // invoke cuda DNN library for Y = softmax(X) - std::vector dims({batch_count, 1, 1, element_count}); + const int64_t dims[]{batch_count, 1, 1, element_count}; const auto alpha = Consts::One; const auto beta = Consts::Zero; CudnnTensor input_tensor, output_tensor; @@ -227,7 +227,7 @@ Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* outpu #define SPECIALIZED_BIAS_SOFTMAX_IMPL(T) \ template Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T * output_data, \ const T* input_data, const T* bias_data, int element_count, int batch_count, \ - bool is_inner_broadcast, int bias_broadcast_size, size_t element_size); + bool is_inner_broadcast, int bias_broadcast_size); // MIOpen doesn't support double so ROCm kernel doesn't have double support for now. SPECIALIZED_BIAS_SOFTMAX_IMPL(float) diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.h b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.h index f8fd292be1..4d8788fbe0 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.h +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.h @@ -13,7 +13,7 @@ namespace cuda { template Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* output_data, const T* input_data, const T* bias_data, int element_count, int batch_count, bool is_inner_broadcast, - int bias_broadcast_size, size_t element_size); + int bias_broadcast_size); } // namespace cuda } // namespace contrib diff --git a/onnxruntime/contrib_ops/cuda/math/complex_mul_impl.cu b/onnxruntime/contrib_ops/cuda/math/complex_mul_impl.cu index e72a83cb20..ca94477114 100644 --- a/onnxruntime/contrib_ops/cuda/math/complex_mul_impl.cu +++ b/onnxruntime/contrib_ops/cuda/math/complex_mul_impl.cu @@ -50,7 +50,7 @@ __global__ void _ElementWiseWithStrideTwo( if (id < N / 2) { CUDA_LONG lhs_index = (lhs_need_compute ? 0 : id); CUDA_LONG rhs_index = (rhs_need_compute ? 0 : id); - // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md + // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/main/docs/Broadcasting.md CUDA_LONG offset = id; #pragma unroll for (auto dim = 0; dim < fdm_output_strides.Capacity(); dim++) { diff --git a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc index 1336aa7aac..c05d6335cd 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc @@ -151,7 +151,9 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { } else { dequant_scale = input_scale * weight_scale; } + // scale back and bias + // TODO(tianleiwu): fuse Dequantize with Add bias and Transpose. ORT_RETURN_IF_ERROR(CudaDequantizeWithBias(Stream(), gemm_buffer_quantized.get(), reinterpret_cast(bias->Data()), @@ -164,32 +166,28 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { Tensor* present_tensor = GetPresent(context, past_tensor, batch_size, head_size, sequence_length, past_sequence_length); size_t workSpaceSize = GetAttentionWorkspaceSize(element_size, batch_size, num_heads_, head_size, sequence_length, past_sequence_length); - auto temp_buffer = GetScratchBuffer(workSpaceSize); - if (!LaunchAttentionKernel( + + auto work_space = GetScratchBuffer(workSpaceSize); + return LaunchAttentionKernel( GetDeviceProp(), Stream(), - reinterpret_cast(gemm_buffer.get()), - nullptr == mask_index ? nullptr : mask_index->Data(), - nullptr == mask_index ? gsl::span() : mask_index->Shape().GetDims(), - output->MutableData(), + cublas, + element_size, batch_size, sequence_length, num_heads_, head_size, - temp_buffer.get(), - cublas, - element_size, - is_unidirectional_, past_sequence_length, + is_unidirectional_, + reinterpret_cast(gemm_buffer.get()), + nullptr, // bias has been added + nullptr == mask_index ? nullptr : mask_index->Data(), + nullptr == mask_index ? gsl::span() : mask_index->Shape().GetDims(), nullptr == past_tensor ? nullptr : past_tensor->Data(), - nullptr, // TODO: support add_qk in quantized attention - nullptr == present_tensor ? nullptr : present_tensor->MutableData())) { - // Get last error to reset it to cudaSuccess. - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + nullptr, // TODO: support add_qk in quantized attention + work_space.get(), + output->MutableData(), + nullptr == present_tensor ? nullptr : present_tensor->MutableData()); } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_common.cuh b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_common.cuh new file mode 100644 index 0000000000..ad3647e2a5 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_common.cuh @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +__device__ inline float ToFloat(const __half h) { return __half2float(h); } + +__device__ inline float ToFloat(const float f) { return f; } + +template +__inline__ __device__ T +WarpReduceSum(T val) { + val += __shfl_xor_sync(0xFFFFFFFF, val, 1); + val += __shfl_xor_sync(0xFFFFFFFF, val, 2); + val += __shfl_xor_sync(0xFFFFFFFF, val, 4); + val += __shfl_xor_sync(0xFFFFFFFF, val, 8); + val += __shfl_xor_sync(0xFFFFFFFF, val, 16); + return val; +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm.cc b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm.cc new file mode 100644 index 0000000000..aba3d4759a --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm.cc @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qordered_layer_norm.h" +#include "qordered_layer_norm_impl.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +ONNX_OPERATOR_KERNEL_EX( + QOrderedLayerNormalization, + kMSDomain, + 1, + kCudaExecutionProvider, + (*KernelDefBuilder::Create()) + .TypeConstraint("Q", DataTypeImpl::GetTensorType()) + .TypeConstraint("S", DataTypeImpl::GetTensorType()) + .TypeConstraint("F", BuildKernelDefConstraints()) + .InputMemoryType(OrtMemTypeCPUInput, 1) // scale_X + .InputMemoryType(OrtMemTypeCPUInput, 4), // scale_Y + QOrderedLayerNormalization); + +QOrderedLayerNormalization::QOrderedLayerNormalization(const OpKernelInfo& op_kernel_info) + : CudaKernel(op_kernel_info) { + ORT_ENFORCE(op_kernel_info.GetAttr("axis", &axis_).IsOK()); + + float tmp_epsilon; + ORT_ENFORCE(op_kernel_info.GetAttr("epsilon", &tmp_epsilon).IsOK()); + epsilon_ = tmp_epsilon; + + ORT_ENFORCE(op_kernel_info.GetAttr("order_X", &order_X_).IsOK()); + + ORT_ENFORCE(op_kernel_info.GetAttr("order_Y", &order_Y_).IsOK()); + + ORT_ENFORCE(order_X_ == 1, "QOrderedLayerNormlalization: Only Row major data ordering is currently supported"); + + ORT_ENFORCE(order_X_ == order_Y_, "QOrderedLayerNormlalization: Input ordering should match the output ordering"); +} + +Status QOrderedLayerNormalization::ComputeInternal(OpKernelContext* ctx) const { + typedef typename ToCudaType::MappedType CudaQ; + + // Inputs + const Tensor* X = ctx->Input(0); + const auto* X_data = reinterpret_cast(X->Data()); + const TensorShape& x_shape = X->Shape(); + ORT_ENFORCE(x_shape.GetDims().size() == 3, + "QOrderedLayerNormlalization: Input shape must be {batch, rows, cols}"); + + const Tensor* scale = ctx->Input(2); + const void* scale_data = scale->DataRaw(); + + const Tensor* bias = ctx->Input(3); + const void* bias_data = (nullptr == bias) ? nullptr : bias->DataRaw(); + + const int64_t axis = HandleNegativeAxis(axis_, x_shape.NumDimensions()); + ORT_ENFORCE(axis == 2, + "QOrderedLayerNormlalization: Implementation only " + "supports on normalizing on innermost axis"); + + unsigned int batch = gsl::narrow(x_shape.GetDims()[0]); + unsigned int rows = gsl::narrow(x_shape.GetDims()[1]); + unsigned int cols = gsl::narrow(x_shape.GetDims()[2]); + + if (cols & 0x03) { + return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, + "QOrderedLayerNormlalization: Cols MUST be a multiple of 4"); + } + + // Outputs + Tensor* Y = ctx->Output(0, x_shape); + + if (x_shape.Size() == 0) { + return Status::OK(); + } + + auto* Y_data = reinterpret_cast(Y->MutableData()); + + const float* scale_x = ctx->Input(1)->Data(); + const float* scale_y = ctx->Input(4)->Data(); + + if (scale->IsDataType()) { + QOrderedLayerNorm(Stream(), GetDeviceProp(), static_cast(order_X_), + X_data, *scale_x, Y_data, *scale_y, static_cast(scale_data), + static_cast(bias_data), + static_cast(epsilon_), batch, rows, cols); + } else { + QOrderedLayerNorm(Stream(), GetDeviceProp(), static_cast(order_X_), + X_data, *scale_x, Y_data, *scale_y, static_cast(scale_data), + static_cast(bias_data), + static_cast(epsilon_), batch, rows, cols); + } + + return Status::OK(); +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm.h b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm.h new file mode 100644 index 0000000000..f66a0597ca --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_kernel.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +class QOrderedLayerNormalization final : public CudaKernel { + public: + explicit QOrderedLayerNormalization(const OpKernelInfo& op_kernel_info); + + Status ComputeInternal(OpKernelContext* ctx) const override; + + private: + double epsilon_; + int64_t axis_; + int64_t order_X_; + int64_t order_Y_; +}; + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm_impl.cu b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm_impl.cu new file mode 100644 index 0000000000..8ba13961f5 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm_impl.cu @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "qordered_common.cuh" +#include "qordered_layer_norm_impl.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +constexpr static unsigned QORDER_LAYERNORM_ROWS_PER_BLOCK = 8; + +__device__ inline int8_t QuantizeFloatS8(const float val, const float inverse_scale) { + float dqval = fmaxf(fminf(127.0f, val * inverse_scale), -128.0f); + return static_cast(__float2int_rn(dqval)); +} + +// TODO: Const Ref ? +__device__ inline char4 QuantizeFloat4Char4(const float4 val4, const float rscale) { + return char4{QuantizeFloatS8(val4.x, rscale), QuantizeFloatS8(val4.y, rscale), + QuantizeFloatS8(val4.z, rscale), QuantizeFloatS8(val4.w, rscale)}; +} + +// TODO: Const Ref ? +__device__ inline int32_t Dp4a_Defined(const char4 input_1, const char4 input_2) { + return static_cast(input_1.x) * static_cast(input_2.x) + + static_cast(input_1.y) * static_cast(input_2.y) + + static_cast(input_1.z) * static_cast(input_2.z) + + static_cast(input_1.w) * static_cast(input_2.w); +} + +template +__global__ void QOrderedLayerNormRowKernel(const int8_t* __restrict__ src, const float src_scale, + int8_t* __restrict__ dst, const float dst_scale, + const T* __restrict__ gamma, const T* __restrict__ beta, const float epsilon, + const unsigned rows, const unsigned cols) { + int32_t sum = 0; + int32_t square_sum = 0; + + unsigned r = blockIdx.x * QORDER_LAYERNORM_ROWS_PER_BLOCK + threadIdx.y; + + if (rows <= r) { + return; + } + + const size_t batch_row_index = static_cast(blockIdx.y) * (rows * cols) + r * cols; + src += batch_row_index; + dst += batch_row_index; + for (unsigned c = threadIdx.x << 2; c < cols; c += 128) { + char4 ch4 = __ldg(reinterpret_cast(src + c)); + sum += (static_cast(ch4.x) + static_cast(ch4.y) + + static_cast(ch4.z) + static_cast(ch4.w)); +#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 610 + square_sum = __dp4a(ch4, ch4, square_sum); +#else + square_sum = Dp4a_Defined(ch4, ch4); +#endif + } + + sum = WarpReduceSum(sum); + square_sum = WarpReduceSum(square_sum); + + const float mean = __double2float_rn(src_scale * (double)sum / cols); + + const float rvar = rsqrtf(src_scale * src_scale * __double2float_rn(static_cast(square_sum) - (static_cast(sum) * static_cast(sum) / static_cast(cols))) / cols + epsilon); + + const float dst_rscale = 1.0f / dst_scale; + + float4 f4; + for (unsigned c = threadIdx.x << 2; c < cols; c += 128) { + char4 ch4 = __ldg(reinterpret_cast(src + c)); + + f4.x = (src_scale * ch4.x - mean) * rvar * ToFloat(gamma[c]); + f4.y = (src_scale * ch4.y - mean) * rvar * ToFloat(gamma[c + 1]); + f4.z = (src_scale * ch4.z - mean) * rvar * ToFloat(gamma[c + 2]); + f4.w = (src_scale * ch4.w - mean) * rvar * ToFloat(gamma[c + 3]); + + if (beta) { + f4.x += ToFloat(beta[c]); + f4.y += ToFloat(beta[c + 1]); + f4.z += ToFloat(beta[c + 2]); + f4.w += ToFloat(beta[c + 3]); + } + + *reinterpret_cast(dst + c) = QuantizeFloat4Char4(f4, dst_rscale); + } +} + +template +void QOrderedLayerNorm(cudaStream_t stream, const cudaDeviceProp& /*device_prop*/, cublasLtOrder_t order, + const int8_t* src, const float src_scale, int8_t* dst, const float dst_scale, + const T* gamma, const T* beta, const float epsilon, + const unsigned batch, const unsigned rows, const unsigned cols) { + // The implementation only supports Row major tensor data ordering for now + ORT_ENFORCE(order == CUBLASLT_ORDER_ROW); + + dim3 threads(32, QORDER_LAYERNORM_ROWS_PER_BLOCK, 1); + + dim3 blocks(static_cast(rows + QORDER_LAYERNORM_ROWS_PER_BLOCK - 1) / QORDER_LAYERNORM_ROWS_PER_BLOCK, + static_cast(batch), 1); + + QOrderedLayerNormRowKernel<<>>(src, src_scale, + dst, dst_scale, gamma, beta, + epsilon, rows, cols); +} + +template void QOrderedLayerNorm(cudaStream_t stream, const cudaDeviceProp& /*device_prop*/, cublasLtOrder_t order, + const int8_t* src, const float src_scale, int8_t* dst, const float dst_scale, + const float* gamma, const float* beta, const float epsilon, + const unsigned batch, const unsigned rows, const unsigned cols); + +template void QOrderedLayerNorm<__half>(cudaStream_t stream, const cudaDeviceProp& /*device_prop*/, cublasLtOrder_t order, + const int8_t* src, const float src_scale, int8_t* dst, const float dst_scale, + const __half* gamma, const __half* beta, const float epsilon, + const unsigned batch, const unsigned rows, const unsigned cols); + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm_impl.h b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm_impl.h new file mode 100644 index 0000000000..1ad8b46a68 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_layer_norm_impl.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +template +void QOrderedLayerNorm( + cudaStream_t stream, const cudaDeviceProp& device_prop, cublasLtOrder_t order, + const int8_t* src, const float src_scale, int8_t* dst, const float dst_scale, + const T* gamma, const T* beta, const float epsilon, + unsigned batch, unsigned rows, unsigned cols); + +} +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul.cc b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul.cc new file mode 100644 index 0000000000..34198b7b9f --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul.cc @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qordered_matmul.h" +#include "qordered_matmul_utils.h" + +#include + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +ONNX_OPERATOR_KERNEL_EX( + QOrderedMatMul, + kMSDomain, + 1, + kCudaExecutionProvider, + (*KernelDefBuilder::Create()) + .TypeConstraint("Q", DataTypeImpl::GetTensorType()) + .TypeConstraint("S", DataTypeImpl::GetTensorType()) + .InputMemoryType(OrtMemTypeCPUInput, 1) // scale_A + .InputMemoryType(OrtMemTypeCPUInput, 3) // scale_B + .InputMemoryType(OrtMemTypeCPUInput, 4) // scale_Y + .InputMemoryType(OrtMemTypeCPUInput, 7), // scale_C + QOrderedMatMul); + +static Status ParseRowMajorTensorMetadata(const Tensor& input_tensor, int64_t& rows, + int64_t& cols, int64_t& batch_count, int64_t& element_count) { + const auto& dims = input_tensor.Shape().GetDims(); + + cols = dims.back(); + rows = (dims.size() <= 1 ? 1LL : dims[dims.size() - 2]); + batch_count = (dims.size() <= 2 + ? 1LL + : std::accumulate(dims.begin(), dims.begin() + (dims.size() - 2), + 1LL, std::multiplies())); + + element_count = cols * rows * batch_count; + + return Status::OK(); +} + +QOrderedMatMul::QOrderedMatMul(const OpKernelInfo& info) : CudaKernel(info) { + ORT_ENFORCE(info.GetAttr("order_A", &order_A_).IsOK()); + + ORT_ENFORCE(info.GetAttr("order_B", &order_B_).IsOK()); + + ORT_ENFORCE(info.GetAttr("order_Y", &order_Y_).IsOK()); + + ORT_ENFORCE(order_B_ == CUBLASLT_ORDER_COL && + order_A_ == CUBLASLT_ORDER_ROW && + order_Y_ == CUBLASLT_ORDER_ROW, + "QOrderedMatMul: Input 1 and output's data ordering should be ROW_MAJOR and " + "Input 2's data ordering should be COL_MAJOR"); +} + +Status QOrderedMatMul::QOrderedMatMul::ComputeInternal(OpKernelContext* context) const { + int64_t rows_A = 0, cols_A = 0, batch_A = 1, elements_A = 0; + int64_t rows_B = 0, cols_B = 0, batch_B = 1, elements_B = 0; + int64_t rows_C = 0, cols_C = 0, batch_C = 1, elements_C = 0; + + const Tensor& tensor_A = *context->Input(0); + const Tensor& tensor_B = *context->Input(2); + + // Support General case only. No broadcasting, is handled now. + ORT_ENFORCE(tensor_A.Shape().NumDimensions() == 2 || tensor_A.Shape().NumDimensions() == 3); + ORT_ENFORCE(tensor_B.Shape().NumDimensions() == 2 || tensor_B.Shape().NumDimensions() == 3); + + ORT_RETURN_IF_ERROR(ParseRowMajorTensorMetadata(tensor_A, rows_A, cols_A, batch_A, elements_A)); + ORT_RETURN_IF_ERROR(ParseRowMajorTensorMetadata(tensor_B, rows_B, cols_B, batch_B, elements_B)); + + const float* scale_A = context->Input(1)->Data(); + const float* scale_B = context->Input(3)->Data(); + const float* scale_Y = context->Input(4)->Data(); + ORT_ENFORCE(*scale_Y > 0.0f && *scale_A > 0.0f && *scale_B > 0.0f); + + const Tensor* tensor_bias = context->Input(5); + ORT_ENFORCE(tensor_bias == nullptr || + (tensor_bias->Shape().NumDimensions() == 1 && tensor_bias->Shape()[0] == cols_B)); + + const float* bias = (tensor_bias == nullptr) ? nullptr : tensor_bias->Data(); + + ORT_ENFORCE(batch_A == batch_B || batch_B == 1, "Batch count for matrix A and matrix B does not match"); + ORT_ENFORCE(cols_A == rows_B, "MatMul shape mis-match"); + + TensorShape output_shape(tensor_A.Shape()); + output_shape[output_shape.NumDimensions() - 1] = cols_B; + + const float zero = 0.0f; + const float* scale_C = &zero; + + const int8_t* C = nullptr; + const Tensor* tensor_C = context->Input(6); + + if (tensor_C != nullptr) { + ORT_ENFORCE(tensor_C->Shape().NumDimensions() == 2 || tensor_C->Shape().NumDimensions() == 3); + ORT_RETURN_IF_ERROR(ParseRowMajorTensorMetadata(*tensor_C, rows_C, cols_C, batch_C, elements_C)); + + ORT_ENFORCE(batch_C == batch_A || batch_C == 1); + ORT_ENFORCE(rows_C == rows_A && cols_C == cols_B); + + const Tensor* tensor_scale_C = context->Input(7); + ORT_ENFORCE(tensor_scale_C != nullptr); + scale_C = tensor_scale_C->Data(); + + C = tensor_C->Data(); + } + + Tensor* tensor_Y = context->Output(0, output_shape); + cublasLtHandle_t cublasLt = CublasLtHandle(); + cudaStream_t stream = Stream(); + auto& device_prop = GetDeviceProp(); + + const float alpha = *scale_A * *scale_B / *scale_Y; + const float beta = *scale_C / *scale_Y; + + ORT_RETURN_IF_ERROR(QOrdered_MatMul(cublasLt, stream, device_prop, + static_cast(batch_A), rows_A, cols_B, cols_A, + &alpha, tensor_A.Data(), tensor_B.Data(), + static_cast(batch_B), bias, + &beta, C, static_cast(batch_C), + tensor_Y->MutableData(), static_cast(order_B_))); + + return Status::OK(); +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul.h b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul.h new file mode 100644 index 0000000000..829cd06625 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_kernel.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +class QOrderedMatMul final : public CudaKernel { + public: + explicit QOrderedMatMul(const OpKernelInfo& info); + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + int64_t order_A_; + int64_t order_B_; + int64_t order_Y_; +}; + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul_utils.cc b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul_utils.cc new file mode 100644 index 0000000000..12021c99e6 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul_utils.cc @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qordered_matmul_utils.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +int64_t CalcLeadingDimensionLt(int64_t rows, int64_t cols, cublasLtOrder_t order) { + switch (order) { + case CUBLASLT_ORDER_ROW: + return cols; + case CUBLASLT_ORDER_COL: + return rows; + // TODO: Support other CUBLASLT ordering + default: + return 0; + } +} + +static Status cublasLtMatMulInt8SetupAlgo(cublasLtHandle_t cublasLt_handle, cublasLtMatmulAlgo_t& algo, + int algo_id, int swizzle, + int custom_option, int tile, int splitk_val, + int reduction_scheme, int stages) { + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoInit(cublasLt_handle, CUBLAS_COMPUTE_32I, CUDA_R_32F, + CUDA_R_8I, CUDA_R_8I, CUDA_R_8I, CUDA_R_8I, algo_id, &algo)); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoConfigSetAttribute(&algo, + CUBLASLT_ALGO_CONFIG_CUSTOM_OPTION, + &(custom_option), sizeof(custom_option))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoConfigSetAttribute(&algo, + CUBLASLT_ALGO_CONFIG_TILE_ID, + &(tile), sizeof(tile))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoConfigSetAttribute(&algo, CUBLASLT_ALGO_CONFIG_SPLITK_NUM, + &(splitk_val), sizeof(splitk_val))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoConfigSetAttribute(&algo, CUBLASLT_ALGO_CONFIG_CTA_SWIZZLING, + &(swizzle), sizeof(swizzle))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoConfigSetAttribute(&algo, CUBLASLT_ALGO_CONFIG_REDUCTION_SCHEME, + &(reduction_scheme), sizeof(int))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulAlgoConfigSetAttribute(&algo, CUBLASLT_ALGO_CONFIG_STAGES_ID, + &(stages), sizeof(stages))); + + return Status::OK(); +} + +static inline std::string AlgoKey(const cudaDeviceProp& /*device_prop*/, + int batch_count, int m, int n, int k, + cublasLtOrder_t weight_order, cublasLtOrder_t input_output_order) { + std::stringstream ss; + ss << batch_count << "-" << m << "_" << n << "_" << k << "-" + << static_cast(weight_order) << "-" << static_cast(input_output_order); + return ss.str(); +} + +CublasLtMMAlgoMap& CublasLtMMAlgoMap::Instance() { + static CublasLtMMAlgoMap instance; + return instance; +} + +void CublasLtMMAlgoMap::GetAlgo(cublasLtHandle_t cublasLt_handle, cublasLtMatmulAlgo_t& algo, + const cudaDeviceProp& device_prop, + int batch_count, int m, int n, int k, + cublasLtOrder_t weight_order, + cublasLtOrder_t input_output_order) const { + ORT_ENFORCE(input_output_order == CUBLASLT_ORDER_ROW, "Input/Output should be ORDER_ROW"); + ORT_ENFORCE(weight_order == CUBLASLT_ORDER_COL, "Weight should be ORDER_COL"); + + const std::string& key = AlgoKey(device_prop, batch_count, m, n, k, weight_order, input_output_order); + + // TODO: Pre-load the best_algos_ from a config file which will be generated offline (or) + // Find the best algo dynamically in the warm-up run and cache it in `best_algos_` + auto algo_it = best_algos_.find(key); + if (algo_it != best_algos_.end() && algo_it->second.workspace_size == 0) { + const auto& algo_info = algo_it->second; + ORT_THROW_IF_ERROR(cublasLtMatMulInt8SetupAlgo(cublasLt_handle, algo, algo_info.algo_id, + algo_info.swizzle, algo_info.custom_option, + algo_info.tile, algo_info.splitk_val, + algo_info.reduction_scheme, algo_info.stages)); + } else { + // Default algo + int algo_id = 21; + int stages = 0; + ORT_THROW_IF_ERROR(cublasLtMatMulInt8SetupAlgo(cublasLt_handle, algo, algo_id, 0, 0, 20, 0, 0, stages)); + } +} + +static Status CreateLtMatrixLayout(cublasLtMatrixLayout_t& layout_desc, + int const batch_count, int64_t const rows_after_op, int64_t const cols_after_op, + cudaDataType_t const data_type, cublasLtOrder_t const mat_order, + cublasOperation_t const mat_trans) { + if (mat_trans == CUBLAS_OP_T) { + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutCreate(&layout_desc, data_type, cols_after_op, rows_after_op, + CalcLeadingDimensionLt(cols_after_op, rows_after_op, mat_order))); + } else { + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutCreate(&layout_desc, data_type, rows_after_op, cols_after_op, + CalcLeadingDimensionLt(rows_after_op, cols_after_op, mat_order))); + } + + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutSetAttribute(layout_desc, + CUBLASLT_MATRIX_LAYOUT_ORDER, + &mat_order, sizeof(mat_order))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutSetAttribute(layout_desc, + CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, + &batch_count, sizeof(batch_count))); + + if (batch_count > 1) { + int64_t batch_stride = rows_after_op * cols_after_op; + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutSetAttribute(layout_desc, CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, + &batch_stride, sizeof(batch_stride))); + } + + return Status::OK(); +} + +Status QOrdered_MatMul(cublasLtHandle_t cublasLt_handle, cudaStream_t stream, + const cudaDeviceProp& device_prop, + int32_t batch_count, int64_t m, int64_t n, int64_t k, + const float* alpha, const int8_t* A, const int8_t* B, int32_t batch_B, + const float* bias, const float* beta, + const int8_t* C, int32_t batch_C, + int8_t* D, cublasLtOrder_t weight_order) { + ORT_ENFORCE(weight_order == CUBLASLT_ORDER_COL, "Weight should be ORDER_COL"); + + const cublasOperation_t transpose_op = CUBLAS_OP_T; + + cublasLtMatmulDesc_t matmul_desc = nullptr; + auto clean_matmul_desc = gsl::finally([&matmul_desc]() {if (matmul_desc) cublasLtMatmulDescDestroy(matmul_desc); }); + + cublasLtMatrixLayout_t desc_A = nullptr; + auto clean_desc_A = gsl::finally([&desc_A]() {if (desc_A) cublasLtMatrixLayoutDestroy(desc_A); }); + + cublasLtMatrixLayout_t desc_B = nullptr; + auto clean_desc_B = gsl::finally([&desc_B]() {if (desc_B) cublasLtMatrixLayoutDestroy(desc_B); }); + + cublasLtMatrixLayout_t desc_C = nullptr; + auto clean_desc_C = gsl::finally([&desc_C]() {if (desc_C) cublasLtMatrixLayoutDestroy(desc_C); }); + + cublasLtMatrixLayout_t desc_D = nullptr; + auto clean_desc_D = gsl::finally([&desc_D]() {if (desc_D) cublasLtMatrixLayoutDestroy(desc_D); }); + + const float beta_zero = 0.0f; + beta = (C == nullptr ? &beta_zero : beta); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulDescCreate(&matmul_desc, CUBLAS_COMPUTE_32I, CUDA_R_32F)); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulDescSetAttribute(matmul_desc, + CUBLASLT_MATMUL_DESC_TRANSA, + &transpose_op, sizeof(transpose_op))); + + if (bias != nullptr) { + cublasLtEpilogue_t epilogue_bias = CUBLASLT_EPILOGUE_BIAS; + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulDescSetAttribute(matmul_desc, + CUBLASLT_MATMUL_DESC_EPILOGUE, + &epilogue_bias, sizeof(epilogue_bias))); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmulDescSetAttribute(matmul_desc, + CUBLASLT_MATMUL_DESC_BIAS_POINTER, + &bias, sizeof(bias))); + } + + ORT_RETURN_IF_ERROR(CreateLtMatrixLayout(desc_A, batch_count, k, m, CUDA_R_8I, + CUBLASLT_ORDER_COL, CUBLAS_OP_N)); // for A' + + ORT_RETURN_IF_ERROR(CreateLtMatrixLayout(desc_B, batch_B, k, n, CUDA_R_8I, + CUBLASLT_ORDER_COL, CUBLAS_OP_N)); // For B + + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutSetAttribute(desc_B, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, + &batch_count, sizeof(batch_count))); + + ORT_RETURN_IF_ERROR(CreateLtMatrixLayout(desc_D, batch_count, n, m, + CUDA_R_8I, CUBLASLT_ORDER_COL, + CUBLAS_OP_N)); // For D' + + if (C != nullptr) { + ORT_RETURN_IF_ERROR(CreateLtMatrixLayout(desc_C, batch_C, n, m, CUDA_R_8I, + CUBLASLT_ORDER_COL, CUBLAS_OP_N)); // For C' + + CUBLAS_RETURN_IF_ERROR(cublasLtMatrixLayoutSetAttribute(desc_C, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, + &batch_count, sizeof(batch_count))); + } + + cublasLtMatmulAlgo_t algo; + CublasLtMMAlgoMap::Instance().GetAlgo(cublasLt_handle, algo, device_prop, batch_count, + static_cast(m), static_cast(n), static_cast(k), + weight_order, CUBLASLT_ORDER_ROW); + + CUBLAS_RETURN_IF_ERROR(cublasLtMatmul(cublasLt_handle, matmul_desc, alpha, + B, desc_B, + A, desc_A, + beta, + C == nullptr ? D : C, C == nullptr ? desc_D : desc_C, + D, desc_D, + &algo, nullptr, 0, // algo, workspace, workspace_size + stream)); + return Status::OK(); +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul_utils.h b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul_utils.h new file mode 100644 index 0000000000..143cdb19dd --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_matmul_utils.h @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +#include +#include + +namespace onnxruntime { + +namespace contrib { +namespace cuda { + +class CublasLtMMAlgoMap { + public: + static CublasLtMMAlgoMap& Instance(); + + void GetAlgo(cublasLtHandle_t cublasLt_handle, cublasLtMatmulAlgo_t& algo, + const cudaDeviceProp& device_prop, + int batch_count, int m, int n, int k, + cublasLtOrder_t weight_order, + cublasLtOrder_t input_output_order = CUBLASLT_ORDER_ROW) const; + + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CublasLtMMAlgoMap); + + private: + CublasLtMMAlgoMap() = default; + + ~CublasLtMMAlgoMap() = default; + + struct CublasLtMatmulAlgoInfo { + int algo_id, custom_option, tile, splitk_val, + swizzle, reduction_scheme, workspace_size, stages; + + float exec_time; + }; + + std::unordered_map best_algos_; +}; + +Status QOrdered_MatMul(cublasLtHandle_t cublasLt_handle, cudaStream_t stream, + const cudaDeviceProp& device_prop, + int32_t batch_count, int64_t m, int64_t n, int64_t k, + const float* alpha, const int8_t* A, const int8_t* B, + int32_t batch_B, + const float* bias, const float* beta, + const int8_t* C, int32_t batch_C, + int8_t* D, cublasLtOrder_t weight_order); + +Status QOrdered_MatMul(cublasLtHandle_t cublasLt_handle, cudaStream_t stream, + const cudaDeviceProp& device_prop, + int32_t batch_count, int64_t m, int64_t n, int64_t k, + const float* alpha, const int8_t* A, const int8_t* B, + const float* bias, int8_t* D, cublasLtOrder_t order_weight); + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops.cc b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops.cc new file mode 100644 index 0000000000..8cb710d0cb --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops.cc @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qordered_unary_ops.h" +#include "qordered_unary_ops_impl.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +ONNX_OPERATOR_KERNEL_EX( + QOrderedGelu, + kMSDomain, + 1, + kCudaExecutionProvider, + (*KernelDefBuilder::Create()) + .TypeConstraint("Q", DataTypeImpl::GetTensorType()) + .TypeConstraint("S", DataTypeImpl::GetTensorType()) + .InputMemoryType(OrtMemTypeCPUInput, 1) // scale_input + .InputMemoryType(OrtMemTypeCPUInput, 2), // scale_output + QOrderedGelu); + +Status QOrderedGelu::ComputeInternal(OpKernelContext* context) const { + const Tensor& input = *context->Input(0); + + const float* scale_input = context->Input(1)->Data(); + const float* scale_output = context->Input(2)->Data(); + + const auto& shape = input.Shape(); + + Tensor* output = context->Output(0, shape); + + QOrderedUnarySharedMemory_Gelu(Stream(), input.Data(), scale_input, output->MutableData(), + scale_output, static_cast(shape.Size())); + + return Status::OK(); +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops.h b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops.h new file mode 100644 index 0000000000..e32e9eb38d --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_kernel.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +class QOrderedGelu final : public CudaKernel { + public: + explicit QOrderedGelu(const OpKernelInfo& info) : CudaKernel(info) {} + + Status ComputeInternal(OpKernelContext* context) const override; +}; + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops_impl.cu b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops_impl.cu new file mode 100644 index 0000000000..95dfc26036 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops_impl.cu @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qordered_unary_ops_impl.h" +#include "core/providers/cuda/cu_inc/common.cuh" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +using namespace onnxruntime::cuda; + +constexpr int kNumLinePerThread = 4; +constexpr int kNumThreadsPerBlock = 256; +constexpr int kNumElementsPerBlockLine = sizeof(char4) * kNumThreadsPerBlock; +constexpr int kNumElementsPerBlock = sizeof(char4) * kNumLinePerThread * kNumThreadsPerBlock; + +template +__global__ void QOrderedUnaryElementWiseSharedMemoryKernel( + const int8_t* input_data, float input_scale, int8_t* output_data, + float inverse_output_scale, const FuncT& functor, CUDA_LONG N) { + __shared__ char table[256]; + + const int calc_id = static_cast(threadIdx.x) - 128; + float gelu_value = inverse_output_scale * functor(input_scale * calc_id); + gelu_value = fmaxf(-128.0f, fmin(127.0f, gelu_value)); + table[threadIdx.x] = static_cast(__float2int_rn(gelu_value)); + __syncthreads(); + + CUDA_LONG id = kNumElementsPerBlock * blockIdx.x + threadIdx.x * (CUDA_LONG)sizeof(char4); + +#pragma unroll + for (int line = 0; line < kNumLinePerThread; line++) { + if (id < N) { + char4 i4 = *(const char4*)(input_data + id); + i4.x = table[128 + i4.x]; + i4.y = table[128 + i4.y]; + i4.z = table[128 + i4.z]; + i4.w = table[128 + i4.w]; + *reinterpret_cast(output_data + id) = i4; + id += kNumElementsPerBlockLine; + } + } +} + +template +void QOrderedUnaryElementWiseSharedMemoryImpl( + cudaStream_t stream, + const int8_t* input_data, + const float* input_scale, + int8_t* output_data, + const float* output_scale, + const FuncT& func, + size_t count) { + if (count > 0) { + float inverse_output_scale = 1.0f / *output_scale; + int blocksPerGrid = static_cast(CeilDiv(count, kNumElementsPerBlock)); + + QOrderedUnaryElementWiseSharedMemoryKernel<<>>( + input_data, *input_scale, output_data, inverse_output_scale, func, static_cast(count)); + } +} + +struct QOrderedUnaryOpGelu { + static constexpr float sqrt2 = 1.4142135623730950488016887242097f; + + __device__ __inline__ float operator()(const float& x) const { + return x * 0.5f * (1.0f + erff(x / sqrt2)); + } +}; + +QORDERED_UNARY_OP_SHARED_MEMORY_DECLARATION(Gelu) { + QOrderedUnaryElementWiseSharedMemoryImpl( + stream, input_data, input_scale, output_data, output_scale, QOrderedUnaryOpGelu(), count); +} + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops_impl.h b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops_impl.h new file mode 100644 index 0000000000..8c1ad09021 --- /dev/null +++ b/onnxruntime/contrib_ops/cuda/quantization/qordered_ops/qordered_unary_ops_impl.h @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_common.h" + +namespace onnxruntime { +namespace contrib { +namespace cuda { + +#define QORDERED_UNARY_OP_SHARED_MEMORY_DECLARATION(name) \ + void QOrderedUnarySharedMemory_##name( \ + cudaStream_t stream, \ + const int8_t* input_data, \ + const float* input_scale, \ + int8_t* output_data, \ + const float* output_scale, \ + size_t count) + +QORDERED_UNARY_OP_SHARED_MEMORY_DECLARATION(Gelu); + +} // namespace cuda +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc index 0432b1a345..3679875a8a 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc +++ b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc @@ -409,16 +409,16 @@ Status ProcessLogits(const OrtValue& logits, // template Status GreedySearchProcessLogits( - const OrtValue& logits, // logits output of subgraph - transformers::IGreedySearchState* greedy_state, // state - transformers::ISequences* sequences, // sequences - AllocatorPtr& allocator, // default allocator - onnxruntime::concurrency::ThreadPool* thread_pool, // thread pool (for CPU only) - transformers::ILogitsProcessorList* logits_processors, // logits processors - const transformers::IBeamSearchParameters* parameters, // parameters - int step, // iteration counter - void* stream, // cuda stream (for CUDA only) - const transformers::IConsoleDumper* dumper) { // tensor dumper + const OrtValue& logits, // logits output of subgraph + transformers::IGreedySearchState* greedy_state, // state + transformers::ISequences* sequences, // sequences + AllocatorPtr& allocator, // default allocator + onnxruntime::concurrency::ThreadPool* thread_pool, // thread pool (for CPU only) + transformers::ILogitsProcessorList* logits_processors, // logits processors + const transformers::IBeamSearchParameters* parameters, // parameters + int step, // iteration counter + void* stream, // cuda stream (for CUDA only) + const transformers::IConsoleDumper* dumper) { // tensor dumper ORT_UNUSED_PARAMETER(logits_processors); #ifndef DEBUG_GENERATION @@ -510,7 +510,7 @@ Status GreedySearchProcessLogits( const Tensor& input = next_token_scores_value.Get(); constexpr int axis = 1; - const unsigned top_k = static_cast(1); + constexpr unsigned top_k = static_cast(1); constexpr bool largest = true; constexpr bool sorted = false; diff --git a/onnxruntime/contrib_ops/rocm/bert/attention.cc b/onnxruntime/contrib_ops/rocm/bert/attention.cc index 8258bf78c6..df4e2bc3bc 100644 --- a/onnxruntime/contrib_ops/rocm/bert/attention.cc +++ b/onnxruntime/contrib_ops/rocm/bert/attention.cc @@ -41,7 +41,8 @@ Status Attention::ComputeInternal(OpKernelContext* context) const { const Tensor* extra_add_qk = context->Input(5); auto& device_prop = GetDeviceProp(); - ORT_RETURN_IF_ERROR(CheckInputs(input->Shape(), weights->Shape(), bias->Shape(), mask_index, past, extra_add_qk, device_prop.maxThreadsPerBlock)); + ORT_RETURN_IF_ERROR(CheckInputs(input->Shape(), weights->Shape(), bias->Shape(), + mask_index, past, extra_add_qk, device_prop.maxThreadsPerBlock)); // input shape (batch_size, sequence_length, input_hidden_size) const auto& shape = input->Shape(); @@ -92,33 +93,29 @@ Status Attention::ComputeInternal(OpKernelContext* context) const { reinterpret_cast(input->Data()), k, &one, reinterpret_cast(gemm_buffer.get()), n)); - size_t workSpaceSize = GetAttentionWorkspaceSize(element_size, batch_size, num_heads_, head_size, sequence_length, past_sequence_length); - auto temp_buffer = GetScratchBuffer(workSpaceSize); - if (!LaunchAttentionKernel( + size_t workSpaceSize = GetAttentionWorkspaceSize(element_size, batch_size, num_heads_, head_size, + sequence_length, past_sequence_length); + + auto work_space = GetScratchBuffer(workSpaceSize); + return LaunchAttentionKernel( device_prop, Stream(), - reinterpret_cast(gemm_buffer.get()), - nullptr == mask_index ? nullptr : mask_index->Data(), - nullptr == mask_index ? gsl::span() : mask_index->Shape().GetDims(), - output->MutableData(), + rocblas, + element_size, batch_size, sequence_length, num_heads_, head_size, - temp_buffer.get(), - rocblas, - element_size, - is_unidirectional_, past_sequence_length, + is_unidirectional_, + reinterpret_cast(gemm_buffer.get()), + nullptr == mask_index ? nullptr : mask_index->Data(), + nullptr == mask_index ? gsl::span() : mask_index->Shape().GetDims(), nullptr == past ? nullptr : past->Data(), nullptr == extra_add_qk ? nullptr : extra_add_qk->Data(), - nullptr == present ? nullptr : present->MutableData())) { - // Get last error to reset it to hipSuccess. - HIP_CALL(hipGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + work_space.get(), + output->MutableData(), + nullptr == present ? nullptr : present->MutableData()); } } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/attention_impl.cu b/onnxruntime/contrib_ops/rocm/bert/attention_impl.cu index c87c12a1f9..315fa7acd2 100644 --- a/onnxruntime/contrib_ops/rocm/bert/attention_impl.cu +++ b/onnxruntime/contrib_ops/rocm/bert/attention_impl.cu @@ -32,10 +32,7 @@ limitations under the License. using namespace onnxruntime::rocm; using namespace hipcub; -#define CHECK_ROCM(expr) \ - if (!HIP_CALL(expr)) { \ - return false; \ - } +#define CHECK_ROCM(expr) HIP_RETURN_IF_ERROR(expr) namespace onnxruntime { namespace contrib { @@ -45,9 +42,12 @@ static size_t AlignTo(size_t a, size_t b) { return CeilDiv(a, b) * b; } -size_t GetAttentionScratchSize(size_t element_size, int batch_size, int num_heads, int sequence_length, int all_sequence_length) { - const size_t len = batch_size * num_heads * sequence_length * all_sequence_length; - const size_t bytes = len * element_size; +size_t GetAttentionScratchSize(size_t element_size, + int batch_size, + int num_heads, + int sequence_length, + int all_sequence_length) { + const size_t bytes = element_size * batch_size * num_heads * sequence_length * all_sequence_length; const size_t alignment = 256; const size_t bytesAligned = AlignTo(bytes, alignment); @@ -61,19 +61,35 @@ size_t GetAttentionWorkspaceSize( int head_size, int sequence_length, int past_sequence_length) { - size_t qkv_size = 3 * batch_size * sequence_length * num_heads * head_size * element_size; - return qkv_size + 2 * GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, past_sequence_length + sequence_length); + size_t qkv_size = element_size * 3 * batch_size * sequence_length * num_heads * head_size; + return qkv_size + 2 * GetAttentionScratchSize(element_size, batch_size, num_heads, + sequence_length, past_sequence_length + sequence_length); } template -bool QkvToContext( - const hipDeviceProp_t& prop, rocblas_handle& rocblas, hipStream_t stream, - const int batch_size, const int sequence_length, const int num_heads, const int head_size, const size_t element_size, - const T* input, T* output, T* workspace, - const int* mask_index, gsl::span mask_index_dims, - bool is_unidirectional, int past_sequence_length, const T* past, const T* extra_add_qk, T* present, bool use_persistent_softmax) { +Status QkvToContext( + const hipDeviceProp_t& prop, + rocblas_handle& rocblas, + hipStream_t stream, + const int batch_size, + const int sequence_length, + const int num_heads, + const int head_size, + const size_t element_size, + const T* input, + T* output, + T* workspace, + const int* mask_index, + gsl::span mask_index_dims, + bool is_unidirectional, + int past_sequence_length, + const T* past, + const T* extra_add_qk, + T* present, + bool use_persistent_softmax) { const int all_sequence_length = past_sequence_length + sequence_length; - const size_t bytes = GetAttentionScratchSize(element_size, batch_size, num_heads, sequence_length, all_sequence_length); + const size_t bytes = GetAttentionScratchSize(element_size, batch_size, num_heads, + sequence_length, all_sequence_length); T* scratch1 = workspace; T* scratch2 = scratch1 + (bytes / element_size); T* scratch3 = scratch2 + (bytes / element_size); @@ -81,9 +97,8 @@ bool QkvToContext( const int max_threads_per_block = prop.maxThreadsPerBlock; // input should be BxSx3xNxH => scratch3: 3xBxNxSxH - if (!LaunchTransQkv(stream, 3, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, false, input, scratch3)) { - return false; - } + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 3, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, false, input, scratch3)); // now scratch3 has Q, K, V: each has size BxNxSxH const int batches = batch_size * num_heads; @@ -101,9 +116,9 @@ bool QkvToContext( // past_v (BxNxS'xH) + v (BxNxSxH) => present_v (BxNxS*xH) const int present_size_per_batch = all_sequence_length * head_size; if (nullptr != present) { - if (!LaunchConcatPastToPresent(stream, all_sequence_length, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, past, k, present)) { - return false; - } + ORT_RETURN_IF_ERROR( + LaunchConcatPastToPresent(stream, all_sequence_length, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, past, k, present)); // update pointers to present_k and present_v. k = present; @@ -120,121 +135,135 @@ bool QkvToContext( typedef typename ToHipType::MappedType HipT; - //float one = 1.0f; - //float zero = 0.f; + // float one = 1.0f; + // float zero = 0.f; const HipT one = ToHipType::FromFloat(1.0f); const HipT zero = ToHipType::FromFloat(0.f); // For raw attention mask, the scalar if 1/sqrt(H) is moved to softmax computation. - //float temp_alpha = use_raw_attention_mask ? one : rsqrt_head_size; + // float temp_alpha = use_raw_attention_mask ? one : rsqrt_head_size; const HipT alpha = use_raw_attention_mask ? one : ToHipType::FromFloat(rsqrt_head_size); - if (!ROCBLAS_CALL(rocblasGemmStridedBatchedHelper( - rocblas, rocblas_operation_transpose, rocblas_operation_none, all_sequence_length, sequence_length, head_size, &alpha, k, head_size, present_size_per_batch, - q, head_size, size_per_batch, &zero, scratch1, all_sequence_length, temp_matrix_size, batches))) { - return false; - } + ROCBLAS_RETURN_IF_ERROR(rocblasGemmStridedBatchedHelper( + rocblas, rocblas_operation_transpose, rocblas_operation_none, + all_sequence_length, sequence_length, head_size, + &alpha, k, head_size, present_size_per_batch, + q, head_size, size_per_batch, + &zero, scratch1, all_sequence_length, temp_matrix_size, batches)); // apply softmax and store result P to scratch2: BxNxSxS* if (use_raw_attention_mask) { // 2d, 3d or 4d attention mask const int mask_dimension = static_cast(mask_index_dims.size()); - const int64_t max_sequence_length = mask_dimension == 4 ? mask_index_dims.at(3) : 0; + const int max_sequence_length = mask_dimension == 4 ? static_cast(mask_index_dims.at(3)) : 0; - T* persistent_softmax_workspace = scratch1; // replace Q*K' in place with masked score if persistent softmax is selected. - if (!ComputeSoftmaxWithRawMask(stream, all_sequence_length, sequence_length, batch_size, num_heads, mask_index, nullptr, extra_add_qk, scratch1, scratch2, - is_unidirectional, rsqrt_head_size, mask_dimension, static_cast(max_sequence_length), - use_persistent_softmax, persistent_softmax_workspace)) { - return false; - } + T* persistent_softmax_workspace = scratch1; // replace Q*K' in place if persistent softmax is selected. + ORT_RETURN_IF_ERROR( + ComputeSoftmaxWithRawMask(stream, all_sequence_length, sequence_length, batch_size, num_heads, + mask_index, nullptr, extra_add_qk, scratch1, scratch2, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax, persistent_softmax_workspace)); } else if (nullptr != mask_index) { // 1d mask index ORT_ENFORCE(mask_index_dims.size() == 1); // mask_index has 1D shape: either (batch_size) or (2*batch_size). Only the later one has start postions. const int* mask_start = (mask_index_dims.at(0) > batch_size) ? mask_index + batch_size : nullptr; - if (!ComputeSoftmaxWithMask1D(stream, all_sequence_length, sequence_length, batch_size, num_heads, mask_index, mask_start, extra_add_qk, scratch1, scratch2, is_unidirectional)) { - return false; - } + ORT_RETURN_IF_ERROR(ComputeSoftmaxWithMask1D(stream, all_sequence_length, sequence_length, batch_size, num_heads, + mask_index, mask_start, extra_add_qk, scratch1, scratch2, is_unidirectional)); } else { // no mask - if (!ComputeSoftmax(stream, all_sequence_length, sequence_length, batch_size, num_heads, extra_add_qk, scratch1, scratch2, is_unidirectional)) { - return false; - } + ORT_RETURN_IF_ERROR(ComputeSoftmax(stream, all_sequence_length, sequence_length, batch_size, num_heads, + extra_add_qk, scratch1, scratch2, is_unidirectional)); } // compute P*V (as V*P), and store in scratch3: BxNxSxH - if (!ROCBLAS_CALL(rocblasGemmStridedBatchedHelper( - rocblas, rocblas_operation_none, rocblas_operation_none, head_size, sequence_length, all_sequence_length, &one, v, head_size, present_size_per_batch, - scratch2, all_sequence_length, temp_matrix_size, &zero, scratch3, head_size, size_per_batch, batches))) { - return false; - } + ROCBLAS_RETURN_IF_ERROR(rocblasGemmStridedBatchedHelper( + rocblas, rocblas_operation_none, rocblas_operation_none, + head_size, sequence_length, all_sequence_length, + &one, v, head_size, present_size_per_batch, + scratch2, all_sequence_length, temp_matrix_size, + &zero, scratch3, head_size, size_per_batch, batches)); // scratch3 is BxNxSxH, transpose to output BxSxNxH - return LaunchTransCtx(stream, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, false, scratch3, output); + return LaunchTransCtx(stream, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, false, scratch3, output); } -bool LaunchAttentionKernel( +Status LaunchAttentionKernel( const hipDeviceProp_t& prop, hipStream_t stream, + rocblas_handle& rocblas, + const size_t element_size, + int batch_size, + int sequence_length, + int num_heads, + int head_size, + int past_sequence_length, + bool is_unidirectional, const void* input, const int* mask_index, gsl::span mask_index_dims, - void* output, - const int batch_size, - const int sequence_length, - const int num_heads, - const int head_size, - void* workspace, - rocblas_handle& rocblas, - const size_t element_size, - bool is_unidirectional, - int past_sequence_length, const void* past, const void* extra_add_qk, + void* workspace, + void* output, void* present) { // For testing, environment variable ORT_TRANSFORMER_OPTIONS=1 could enable persistent softmax const TransformerOptions* options = TransformerOptions::GetInstance(); bool use_persistent_softmax = options->IsPrecisionMode() && !options->DisablePersistentSoftmax(); if (element_size == 2) { - return QkvToContext(prop, rocblas, stream, - batch_size, sequence_length, num_heads, head_size, element_size, - reinterpret_cast(input), reinterpret_cast<__half*>(output), reinterpret_cast<__half*>(workspace), - mask_index, mask_index_dims, is_unidirectional, - past_sequence_length, reinterpret_cast(past), reinterpret_cast(extra_add_qk), - reinterpret_cast<__half*>(present), use_persistent_softmax); + return QkvToContext( + prop, rocblas, stream, batch_size, sequence_length, num_heads, head_size, element_size, + reinterpret_cast(input), + reinterpret_cast<__half*>(output), + reinterpret_cast<__half*>(workspace), + mask_index, + mask_index_dims, + is_unidirectional, + past_sequence_length, + reinterpret_cast(past), + reinterpret_cast(extra_add_qk), + reinterpret_cast<__half*>(present), + use_persistent_softmax); } else { - return QkvToContext(prop, rocblas, stream, - batch_size, sequence_length, num_heads, head_size, element_size, - reinterpret_cast(input), reinterpret_cast(output), reinterpret_cast(workspace), - mask_index, mask_index_dims, is_unidirectional, - past_sequence_length, reinterpret_cast(past), reinterpret_cast(extra_add_qk), - reinterpret_cast(present), use_persistent_softmax); + return QkvToContext( + prop, rocblas, stream, batch_size, sequence_length, num_heads, head_size, element_size, + reinterpret_cast(input), + reinterpret_cast(output), + reinterpret_cast(workspace), + mask_index, + mask_index_dims, + is_unidirectional, + past_sequence_length, + reinterpret_cast(past), + reinterpret_cast(extra_add_qk), + reinterpret_cast(present), + use_persistent_softmax); } } - template -bool DecoderQkvToContext( - const hipDeviceProp_t& prop, - hipStream_t stream, - rocblas_handle& rocblas, - const size_t element_size, - const int batch_size, - const int sequence_length, - const int kv_sequence_length, - const int num_heads, - const int head_size, - const bool static_kv, - const bool use_past, - const bool has_layer_state, - const bool has_key_padding_mask, - const T* gemm_query_buffer, - const T* gemm_kv_buffer, - const bool* key_padding_mask, - const T* key_cache, - const T* value_cache, - T* qkv_buffer, - T* workspace_buffer, - T* output, - T* new_key_cache, - T* new_value_cache) { +Status DecoderQkvToContext( + const hipDeviceProp_t& prop, + hipStream_t stream, + rocblas_handle& rocblas, + const size_t element_size, + const int batch_size, + const int sequence_length, + const int kv_sequence_length, + const int num_heads, + const int head_size, + const bool static_kv, + const bool use_past, + const bool has_layer_state, + const bool has_key_padding_mask, + const T* gemm_query_buffer, + const T* gemm_kv_buffer, + const bool* key_padding_mask, + const T* key_cache, + const T* value_cache, + T* qkv_buffer, + T* workspace_buffer, + T* output, + T* new_key_cache, + T* new_value_cache) { const int max_threads_per_block = prop.maxThreadsPerBlock; const int BN = batch_size * num_heads; const int BHN = BN * head_size; @@ -246,48 +275,40 @@ bool DecoderQkvToContext( const T* q = qkv_buffer; // transpose q and copy them to qkv_buffer - if (!LaunchTransQkv(stream, 1, sequence_length, batch_size, head_size, - num_heads, max_threads_per_block, true, gemm_query_buffer, qkv_buffer)) { - return false; - } + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, sequence_length, batch_size, head_size, + num_heads, max_threads_per_block, true, gemm_query_buffer, qkv_buffer)); const T* k = qkv_buffer + k_buffer_offset; const T* v = qkv_buffer + v_buffer_offset; if (!has_layer_state || !use_past) { if (!static_kv) { // transpose kv and copy them to qkv_buffer - if (!LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, - max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)) { - return false; - } + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)); } else { // transpose kv and copy them to qkv_buffer - if (!LaunchTransQkv(stream, 2, kv_sequence_length, batch_size, head_size, num_heads, - max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)) { - return false; - } + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 2, kv_sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_kv_buffer, qkv_buffer + k_buffer_offset)); } } else { if (!static_kv) { // transpose kv and copy them to temp_buffer - if (!LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, - max_threads_per_block, true, gemm_kv_buffer, temp_qkv_buffer)) { - return false; - } + ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 2, sequence_length, batch_size, head_size, num_heads, + max_threads_per_block, true, gemm_kv_buffer, temp_qkv_buffer)); // concat cache-k with k and copy to qkv_buffer - if (nullptr != key_cache && !LaunchConcatTensorToTensor(stream, kv_sequence_length, sequence_length, + if (nullptr != key_cache) { + ORT_RETURN_IF_ERROR(LaunchConcatTensorToTensor(stream, kv_sequence_length, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, 1, key_cache, - temp_qkv_buffer, qkv_buffer + k_buffer_offset)) { - return false; + temp_qkv_buffer, qkv_buffer + k_buffer_offset)); } // concat cache-v with v and copy to qkv_buffer - if (nullptr != value_cache && !LaunchConcatTensorToTensor(stream, kv_sequence_length, sequence_length, + if (nullptr != value_cache) { + ORT_RETURN_IF_ERROR(LaunchConcatTensorToTensor(stream, kv_sequence_length, sequence_length, batch_size, head_size, num_heads, max_threads_per_block, 1, value_cache, temp_qkv_buffer + k_buffer_offset, - qkv_buffer + v_buffer_offset)) { - return false; + qkv_buffer + v_buffer_offset)); } } } @@ -324,49 +345,45 @@ bool DecoderQkvToContext( const int strideA = kv_sequence_length * head_size; const int strideB = sequence_length * head_size; if (use_past && static_kv) { - if (!ROCBLAS_CALL(rocblasGemmStridedBatchedHelper( - rocblas, rocblas_operation_transpose, rocblas_operation_none, - kv_sequence_length, sequence_length, head_size, &alpha, key_cache, head_size, strideA, - q, head_size, strideB, &zero, scratch1, kv_sequence_length, temp_matrix_size, BN))) { - return false; - } + ROCBLAS_RETURN_IF_ERROR(rocblasGemmStridedBatchedHelper( + rocblas, rocblas_operation_transpose, rocblas_operation_none, + kv_sequence_length, sequence_length, head_size, + &alpha, key_cache, head_size, strideA, + q, head_size, strideB, + &zero, scratch1, kv_sequence_length, temp_matrix_size, BN)); } else { - if (!ROCBLAS_CALL(rocblasGemmStridedBatchedHelper( - rocblas, rocblas_operation_transpose, rocblas_operation_none, - kv_sequence_length, sequence_length, head_size, &alpha, k, head_size, strideA, - q, head_size, strideB, &zero, scratch1, kv_sequence_length, temp_matrix_size, BN))) { - return false; - } + ROCBLAS_RETURN_IF_ERROR(rocblasGemmStridedBatchedHelper( + rocblas, rocblas_operation_transpose, rocblas_operation_none, + kv_sequence_length, sequence_length, head_size, + &alpha, k, head_size, strideA, + q, head_size, strideB, + &zero, scratch1, kv_sequence_length, temp_matrix_size, BN)); } if (has_key_padding_mask) { - if (!ComputeSoftmaxWithRawMask(stream, kv_sequence_length, sequence_length, batch_size, + ORT_RETURN_IF_ERROR(ComputeSoftmaxWithRawMask(stream, kv_sequence_length, sequence_length, batch_size, num_heads, nullptr, key_padding_mask, nullptr, scratch1, scratch2, - false, 1, 2, static_cast(0), false, nullptr)) { - return false; - } + false, 1, 2, static_cast(0), false, nullptr)); } else { - if (!ComputeSoftmax(stream, kv_sequence_length, sequence_length, batch_size, - num_heads, nullptr, scratch1, scratch2, false)) { - return false; - } + ORT_RETURN_IF_ERROR(ComputeSoftmax(stream, kv_sequence_length, sequence_length, batch_size, + num_heads, nullptr, scratch1, scratch2, false)); } // compute P*V (as V*P), and store in scratch3: BxNxSxH if (use_past && static_kv) { - if (!ROCBLAS_CALL(rocblasGemmStridedBatchedHelper( - rocblas, rocblas_operation_none, rocblas_operation_none, - head_size, sequence_length, kv_sequence_length, &one, value_cache, head_size, strideA, - scratch2, kv_sequence_length, temp_matrix_size, &zero, scratch3, head_size, strideB, BN))) { - return false; - } + ROCBLAS_RETURN_IF_ERROR(rocblasGemmStridedBatchedHelper( + rocblas, rocblas_operation_none, rocblas_operation_none, + head_size, sequence_length, kv_sequence_length, + &one, value_cache, head_size, strideA, + scratch2, kv_sequence_length, temp_matrix_size, + &zero, scratch3, head_size, strideB, BN)); } else { - if (!ROCBLAS_CALL(rocblasGemmStridedBatchedHelper( - rocblas, rocblas_operation_none, rocblas_operation_none, - head_size, sequence_length, kv_sequence_length, &one, v, head_size, strideA, - scratch2, kv_sequence_length, temp_matrix_size, &zero, scratch3, head_size, strideB, BN))) { - return false; - } + ROCBLAS_RETURN_IF_ERROR(rocblasGemmStridedBatchedHelper( + rocblas, rocblas_operation_none, rocblas_operation_none, + head_size, sequence_length, kv_sequence_length, + &one, v, head_size, strideA, + scratch2, kv_sequence_length, temp_matrix_size, + &zero, scratch3, head_size, strideB, BN)); } // scratch3 is BxNxSxH, transpose to output SxBxNxH @@ -374,81 +391,80 @@ bool DecoderQkvToContext( num_heads, max_threads_per_block, true, scratch3, output); } - -bool LaunchDecoderAttentionKernel( - const hipDeviceProp_t& prop, - hipStream_t stream, - rocblas_handle& rocblas, - const size_t element_size, - const int batch_size, - const int sequence_length, - const int kv_sequence_length, - const int num_heads, - const int head_size, - const bool static_kv, - const bool use_past, - const bool has_layer_state, - const bool has_key_padding_mask, - const void* gemm_query_buffer, - const void* gemm_kv_buffer, - const bool* key_padding_mask, - const void* key_cache, - const void* value_cache, - void* qkv_buffer, - void* workspace_buffer, - void* output, - void* new_key_cache, - void* new_value_cache) { +Status LaunchDecoderAttentionKernel( + const hipDeviceProp_t& prop, + hipStream_t stream, + rocblas_handle& rocblas, + const size_t element_size, + const int batch_size, + const int sequence_length, + const int kv_sequence_length, + const int num_heads, + const int head_size, + const bool static_kv, + const bool use_past, + const bool has_layer_state, + const bool has_key_padding_mask, + const void* gemm_query_buffer, + const void* gemm_kv_buffer, + const bool* key_padding_mask, + const void* key_cache, + const void* value_cache, + void* qkv_buffer, + void* workspace_buffer, + void* output, + void* new_key_cache, + void* new_value_cache) { if (element_size == 2) { return DecoderQkvToContext( - prop, - stream, - rocblas, - element_size, - batch_size, - sequence_length, - kv_sequence_length, - num_heads, - head_size, - static_kv, - use_past, - has_layer_state, - has_key_padding_mask, - reinterpret_cast(gemm_query_buffer), - reinterpret_cast(gemm_kv_buffer), - key_padding_mask, - reinterpret_cast(key_cache), - reinterpret_cast(value_cache), - reinterpret_cast(qkv_buffer), - reinterpret_cast(workspace_buffer), - reinterpret_cast(output), - reinterpret_cast(new_key_cache), - reinterpret_cast(new_value_cache)); + prop, + stream, + rocblas, + element_size, + batch_size, + sequence_length, + kv_sequence_length, + num_heads, + head_size, + static_kv, + use_past, + has_layer_state, + has_key_padding_mask, + reinterpret_cast(gemm_query_buffer), + reinterpret_cast(gemm_kv_buffer), + key_padding_mask, + reinterpret_cast(key_cache), + reinterpret_cast(value_cache), + reinterpret_cast(qkv_buffer), + reinterpret_cast(workspace_buffer), + reinterpret_cast(output), + reinterpret_cast(new_key_cache), + reinterpret_cast(new_value_cache)); } else { return DecoderQkvToContext( - prop, - stream, - rocblas, - element_size, - batch_size, - sequence_length, - kv_sequence_length, - num_heads, - head_size, - static_kv, - use_past, - has_layer_state, - has_key_padding_mask, - reinterpret_cast(gemm_query_buffer), - reinterpret_cast(gemm_kv_buffer), - key_padding_mask, - reinterpret_cast(key_cache), - reinterpret_cast(value_cache), - reinterpret_cast(qkv_buffer), - reinterpret_cast(workspace_buffer), - reinterpret_cast(output), - reinterpret_cast(new_key_cache), - reinterpret_cast(new_value_cache)); + prop, + stream, + rocblas, + element_size, + batch_size, + sequence_length, + kv_sequence_length, + num_heads, + head_size, + static_kv, + use_past, + has_layer_state, + has_key_padding_mask, + reinterpret_cast(gemm_query_buffer), + reinterpret_cast(gemm_kv_buffer), + key_padding_mask, + reinterpret_cast(key_cache), + reinterpret_cast(value_cache), + reinterpret_cast(qkv_buffer), + reinterpret_cast(workspace_buffer), + reinterpret_cast(output), + reinterpret_cast(new_key_cache), + reinterpret_cast(new_value_cache)); } } diff --git a/onnxruntime/contrib_ops/rocm/bert/attention_impl.h b/onnxruntime/contrib_ops/rocm/bert/attention_impl.h index 7a17c2c38b..b862280ba0 100644 --- a/onnxruntime/contrib_ops/rocm/bert/attention_impl.h +++ b/onnxruntime/contrib_ops/rocm/bert/attention_impl.h @@ -10,180 +10,181 @@ namespace onnxruntime { namespace contrib { namespace rocm { -size_t GetAttentionScratchSize(size_t element_size, int batch_size, - int num_heads, int sequence_length, int all_sequence_length); +size_t GetAttentionScratchSize( + size_t element_size, + int batch_size, + int num_heads, + int sequence_length, + int all_sequence_length); size_t GetAttentionWorkspaceSize( size_t element_size, - int batchsize, + int batch_size, int num_heads, int head_size, int sequence_length, int past_sequence_length); -bool LaunchAttentionKernel( - const hipDeviceProp_t& prop, // Device Properties - hipStream_t stream, // rocm stream - const void* input, // Input tensor - const int* mask_index, // Attention mask raw data or index - // (end position of each sequence, - // or end positions and start positions). - // NULL means no mask. - gsl::span mask_index_dims, // Mask index shape - void* output, // Output tensor - int batch_size, // Batch size (B) - int sequence_length, // Sequence length (S) - int num_heads, // Number of attention heads (N) - int head_size, // Hidden layer size per head (H) - void* workspace, // Temporary buffer - rocblas_handle& rocblas, // Rocblas handle - const size_t element_size, // Element size of input tensor - bool is_unidirectional, // Whether there is unidirecitonal mask. - int past_sequence_length, // Sequence length in past state - const void* past, // Past state input - const void* extra_add_qk, // Additional Add - void* present // Present state output +Status LaunchAttentionKernel( + const hipDeviceProp_t& prop, // Device Properties + hipStream_t stream, // cuda stream + rocblas_handle& rocblas, // Rocblas handle + const size_t element_size, // Element size of input tensor + int batch_size, // Batch size (B) + int sequence_length, // Sequence length (S) + int num_heads, // Number of attention heads (N) + int head_size, // Hidden layer size per head (H) + int past_sequence_length, // Sequence length in past state + bool is_unidirectional, // Whether there is unidirectional mask. + const void* input, // Input tensor + const int* mask_index, // Attention mask raw data or index. NULL means no mask. + gsl::span mask_index_dims, // Mask index shape + const void* past, // Past state input + const void* extra_add_qk, // Additional Add + void* workspace, // Temporary buffer + void* output, // Output tensor + void* present // Present state output ); -bool LaunchDecoderAttentionKernel( - const hipDeviceProp_t& prop, // Device Properties - hipStream_t stream, // Cuda stream - rocblas_handle& rocblas, // Rocblas handle - const size_t element_size, // Element size of input tensor - const int batch_size, // Batch size (B) - const int sequence_length, // Sequence length (S) - const int kv_sequence_length, // Key/Value/Cache sequence length - const int num_heads, // Number of attention heads (N) - const int head_size, // Hidden layer size per head (H) - const bool static_kv, // Whether cross attention or not - const bool use_past, // Whether use cache or not - const bool has_layer_state, // Whether output cache or not - const bool has_key_padding_mask, // Whether use key_padding_mask or not - const void* gemm_query_buffer, // Query buffer - const void* gemm_kv_buffer, // Key and value buffer - const bool* key_padding_mask, // Key padding mask - const void* key_cache, // Input key cache - const void* value_cache, // Input value cache - void* qkv_buffer, // Temporary buffer - void* workspace_buffer, // Temporary buffer - void* output, // Output tensor - void* new_key_cache, // New_key_cache tensor - void* new_value_cache // New_value_cache tensor +Status LaunchDecoderAttentionKernel( + const hipDeviceProp_t& prop, // Device Properties + hipStream_t stream, // Cuda stream + rocblas_handle& rocblas, // Rocblas handle + const size_t element_size, // Element size of input tensor + const int batch_size, // Batch size (B) + const int sequence_length, // Sequence length (S) + const int kv_sequence_length, // Key/Value/Cache sequence length + const int num_heads, // Number of attention heads (N) + const int head_size, // Hidden layer size per head (H) + const bool static_kv, // Whether cross attention or not + const bool use_past, // Whether use cache or not + const bool has_layer_state, // Whether output cache or not + const bool has_key_padding_mask, // Whether use key_padding_mask or not + const void* gemm_query_buffer, // Query buffer + const void* gemm_kv_buffer, // Key and value buffer + const bool* key_padding_mask, // Key padding mask + const void* key_cache, // Input key cache + const void* value_cache, // Input value cache + void* qkv_buffer, // Temporary buffer + void* workspace_buffer, // Temporary buffer + void* output, // Output tensor + void* new_key_cache, // New_key_cache tensor + void* new_value_cache // New_value_cache tensor ); -bool LaunchTransCtx(hipStream_t stream, - const int sequence_length, const int batch_size, const int head_size, const int num_heads, - const int max_threads_per_block, const bool reversed_bs, const float* input, float* output); +Status LaunchTransCtx(hipStream_t stream, + const int sequence_length, const int batch_size, const int head_size, const int num_heads, + const int max_threads_per_block, const bool reversed_bs, const float* input, float* output); -bool LaunchTransCtx(hipStream_t stream, - const int sequence_length, const int batch_size, const int head_size, const int num_heads, - const int max_threads_per_block, const bool reversed_bs, const half* input, half* output); +Status LaunchTransCtx(hipStream_t stream, + const int sequence_length, const int batch_size, const int head_size, const int num_heads, + const int max_threads_per_block, const bool reversed_bs, const half* input, half* output); -bool LaunchTransQkv(hipStream_t stream, const int matrix_num, - const int sequence_length, const int batch_size, const int head_size, const int num_heads, - const int max_threads_per_block, const bool reversed_bs, const float* input, float* output); +Status LaunchTransQkv(hipStream_t stream, const int matrix_num, + const int sequence_length, const int batch_size, const int head_size, const int num_heads, + const int max_threads_per_block, const bool reversed_bs, const float* input, float* output); -bool LaunchTransQkv(hipStream_t stream, const int matrix_num, - const int sequence_length, const int batch_size, const int head_size, const int num_heads, - const int max_threads_per_block, const bool reversed_bs, const half* input, half* output); +Status LaunchTransQkv(hipStream_t stream, const int matrix_num, + const int sequence_length, const int batch_size, const int head_size, const int num_heads, + const int max_threads_per_block, const bool reversed_bs, const half* input, half* output); -bool LaunchConcatTensorToTensor(hipStream_t stream, - const int all_sequence_length, - const int sequence_length, - const int batch_size, - const int head_size, - const int num_heads, - const int max_threads_per_block, - const int matrix_num, - const float* tensor_in, - const float* tensor_add, - float* tensor_out); +Status LaunchConcatTensorToTensor(hipStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int head_size, + const int num_heads, + const int max_threads_per_block, + const int matrix_num, + const float* tensor_in, + const float* tensor_add, + float* tensor_out); -bool LaunchConcatTensorToTensor(hipStream_t stream, - const int all_sequence_length, - const int sequence_length, - const int batch_size, - const int head_size, - const int num_heads, - const int max_threads_per_block, - const int matrix_num, - const half* tensor_in, - const half* tensor_add, - half* tensor_out); +Status LaunchConcatTensorToTensor(hipStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int head_size, + const int num_heads, + const int max_threads_per_block, + const int matrix_num, + const half* tensor_in, + const half* tensor_add, + half* tensor_out); -bool LaunchConcatPastToPresent(hipStream_t stream, - const int all_sequence_length, - const int sequence_length, - const int batch_size, - const int head_size, - const int num_heads, - const int max_threads_per_block, - const float* past, - const float* k_v, - float* present); +Status LaunchConcatPastToPresent(hipStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int head_size, + const int num_heads, + const int max_threads_per_block, + const float* past, + const float* k_v, + float* present); -bool LaunchConcatPastToPresent(hipStream_t stream, - const int all_sequence_length, - const int sequence_length, - const int batch_size, - const int head_size, - const int num_heads, - const int max_threads_per_block, - const half* past, - const half* k_v, - half* present); +Status LaunchConcatPastToPresent(hipStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int head_size, + const int num_heads, + const int max_threads_per_block, + const half* past, + const half* k_v, + half* present); -inline rocblas_status _compat_rocblas_gemm_strided_batched_ex(rocblas_handle handle, - rocblas_operation transa, - rocblas_operation transb, - int m, - int n, - int k, - const void* alpha, - const void* A, - rocblas_datatype a_type, - rocblas_int lda, - rocblas_stride stride_A, - const void* b, - rocblas_datatype b_type, - rocblas_int ldb, - rocblas_stride stride_b, - const void* beta, - void* c, - rocblas_datatype c_type, - rocblas_int ldc, - rocblas_stride stride_c, - rocblas_int batch_count, - rocblas_datatype compute_type, - rocblas_gemm_algo algo) { - return rocblas_gemm_strided_batched_ex(handle, - transa, - transb, - m, // m - n, // n - k, // k - alpha, // alpha - A, // A - a_type, // A type - lda, // lda - stride_A, // strideA - b, // B - b_type, // B type - ldb, // ldb - stride_b, // strideB - beta, // beta - c, // C - c_type, // C type - ldc, // ldc - stride_c, // strideC - c, // D = C - c_type, // D type = C type - ldc, // ldd = ldc - stride_c, // strideD = strideC - batch_count, // batch count - compute_type, - algo, - 0, 0); +inline rocblas_status _compat_rocblas_gemm_strided_batched_ex(rocblas_handle handle, + rocblas_operation transa, + rocblas_operation transb, + int m, + int n, + int k, + const void* alpha, + const void* A, + rocblas_datatype a_type, + rocblas_int lda, + rocblas_stride stride_A, + const void* b, + rocblas_datatype b_type, + rocblas_int ldb, + rocblas_stride stride_b, + const void* beta, + void* c, + rocblas_datatype c_type, + rocblas_int ldc, + rocblas_stride stride_c, + rocblas_int batch_count, + rocblas_datatype compute_type, + rocblas_gemm_algo algo) { + return rocblas_gemm_strided_batched_ex(handle, + transa, + transb, + m, // m + n, // n + k, // k + alpha, // alpha + A, // A + a_type, // A type + lda, // lda + stride_A, // strideA + b, // B + b_type, // B type + ldb, // ldb + stride_b, // strideB + beta, // beta + c, // C + c_type, // C type + ldc, // ldc + stride_c, // strideC + c, // D = C + c_type, // D type = C type + ldc, // ldd = ldc + stride_c, // strideD = strideC + batch_count, // batch count + compute_type, + algo, + 0, 0); } // Compatible for CublasMathModeSetter diff --git a/onnxruntime/contrib_ops/rocm/bert/attention_softmax.h b/onnxruntime/contrib_ops/rocm/bert/attention_softmax.h index cbd7472a17..4e2bd238bb 100644 --- a/onnxruntime/contrib_ops/rocm/bert/attention_softmax.h +++ b/onnxruntime/contrib_ops/rocm/bert/attention_softmax.h @@ -60,7 +60,9 @@ __device__ inline void Softmax(const int all_sequence_length, for (int i = threadIdx.x; i < valid_end; i += TPB) { if (i >= valid_start) { const int index = offset + i; - float input_at_idx = add_before_softmax == nullptr ? float(input[index]) : float(input[index] + add_before_softmax[index]); + float input_at_idx = add_before_softmax == nullptr + ? static_cast(input[index]) + : static_cast(input[index] + add_before_softmax[index]); if (thread_data_max < input_at_idx) { thread_data_max = input_at_idx; } @@ -92,7 +94,9 @@ __device__ inline void Softmax(const int all_sequence_length, for (int i = threadIdx.x; i < all_sequence_length; i += TPB) { const int index = offset + i; - float input_at_idx = add_before_softmax == nullptr ? float(input[index]) : float(input[index] + add_before_softmax[index]); + float input_at_idx = add_before_softmax == nullptr + ? static_cast(input[index]) + : static_cast(input[index] + add_before_softmax[index]); const float val = (i >= valid_start && i < valid_end) ? expf(input_at_idx - max_block) * sum_reverse_block : 0.f; output[index] = T(val); } @@ -124,7 +128,8 @@ __device__ inline void SoftmaxSmall(const int all_sequence_length, if (is_unidirectional) { int end_unid = all_sequence_length - sequence_length + (blockIdx.x % sequence_length) + 1; if (end_unid <= valid_start) { - // In this situation, mask of [0, end_unid) and [valid_start, valid_end) has -10000, and [end_unid, valid_start) and [valid_end, all_seq_len) has -20000. + // In this situation, mask of [0, end_unid) and [valid_start, valid_end) has -10000, + // and [end_unid, valid_start) and [valid_end, all_seq_len) has -20000. // So [0, end_unid) will also have value after softmax. is_valid = threadIdx.x < end_unid; } else { @@ -138,7 +143,9 @@ __device__ inline void SoftmaxSmall(const int all_sequence_length, // Infinity divided by Infinity is a NAN. Thus, softmax gets a NAN if one or more item are large enough. // a math transform as below is leveraged to get a stable softmax: // e^xi/(e^x1 + ...e^xn) = e^(xi - max) / (e^(x1 - max) + ... + e^(xn - max)) - float input_data = add_before_softmax == nullptr ? float(input[index]) : float(input[index] + add_before_softmax[index]); + float input_data = add_before_softmax == nullptr + ? static_cast(input[index]) + : static_cast(input[index] + add_before_softmax[index]); float thread_data_max = is_valid ? input_data : float(-ROCMRT_INF_F); const auto max = BlockReduce(tmp_storage).Reduce(thread_data_max, hipcub::Max(), end); @@ -199,7 +206,7 @@ __device__ inline void SoftmaxWithRawMaskSmall(const int all_sequence_length, const int sequence_index = blockIdx.x % sequence_length; if (is_unidirectional) { - int from_index = all_sequence_length - sequence_length + sequence_index; // offset of from token in all sequence length. + int from_index = all_sequence_length - sequence_length + sequence_index; // offset in all sequence length. if (threadIdx.x > from_index) { thread_data = -10000.0f; } @@ -212,7 +219,8 @@ __device__ inline void SoftmaxWithRawMaskSmall(const int all_sequence_length, } else if (mask_dimension == 3) { mask_offset = (batch_index * sequence_length + sequence_index) * all_sequence_length + threadIdx.x; } else if (mask_dimension == 4) { - mask_offset = (batch_index * max_sequence_length + all_sequence_length - sequence_length + sequence_index) * max_sequence_length + threadIdx.x; + int from_index = all_sequence_length - sequence_length + sequence_index; + mask_offset = (batch_index * max_sequence_length + from_index) * max_sequence_length + threadIdx.x; } if (nullptr == key_padding_mask) { @@ -257,41 +265,53 @@ __device__ inline void SoftmaxWithRawMaskSmall(const int all_sequence_length, } template -__global__ void SoftmaxKernelSmall(const int all_sequence_length, const int sequence_length, const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { - SoftmaxSmall(all_sequence_length, sequence_length, all_sequence_length, 0, add_before_softmax, input, output, is_unidirectional); +__global__ void SoftmaxKernelSmall(const int all_sequence_length, const int sequence_length, + const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { + SoftmaxSmall(all_sequence_length, sequence_length, all_sequence_length, 0, + add_before_softmax, input, output, is_unidirectional); } template -__global__ void SoftmaxKernel(const int all_sequence_length, const int sequence_length, const T* add_before_softmax, const T* input, T* output) { - Softmax(all_sequence_length, sequence_length, all_sequence_length, 0, add_before_softmax, input, output); +__global__ void SoftmaxKernel(const int all_sequence_length, const int sequence_length, + const T* add_before_softmax, const T* input, T* output) { + Softmax(all_sequence_length, sequence_length, all_sequence_length, 0, + add_before_softmax, input, output); } template -bool ComputeSoftmax( - hipStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, +Status ComputeSoftmax( + hipStream_t stream, + const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { const dim3 grid(sequence_length * num_heads, batch_size, 1); if (all_sequence_length <= 32) { const int blockSize = 32; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 64) { const int blockSize = 64; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 128) { const int blockSize = 128; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 256) { const int blockSize = 256; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 512) { const int blockSize = 512; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 1024) { const int blockSize = 1024; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output, is_unidirectional); } else if (!is_unidirectional) { const int blockSize = 1024; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, add_before_softmax, input, output); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, add_before_softmax, input, output); } else { ORT_THROW("Attention ROCM operator does not support total sequence length > 1024."); } @@ -300,7 +320,10 @@ bool ComputeSoftmax( } template -__global__ void MaskedSoftmaxKernelSmall(const int all_sequence_length, const int sequence_length, const int* mask_end, const int* mask_start, const T* add_before_softmax, const T* input, T* output, bool is_unidirectional) { +__global__ void MaskedSoftmaxKernelSmall(const int all_sequence_length, const int sequence_length, + const int* mask_end, const int* mask_start, + const T* add_before_softmax, const T* input, T* output, + bool is_unidirectional) { __shared__ int start_position; __shared__ int end_position; @@ -317,11 +340,14 @@ __global__ void MaskedSoftmaxKernelSmall(const int all_sequence_length, const in } __syncthreads(); - SoftmaxSmall(all_sequence_length, sequence_length, end_position, start_position, add_before_softmax, input, output, is_unidirectional); + SoftmaxSmall(all_sequence_length, sequence_length, end_position, start_position, + add_before_softmax, input, output, is_unidirectional); } template -__global__ void MaskedSoftmaxKernel(const int all_sequence_length, const int sequence_length, const int* mask_end, const int* mask_start, const T* add_before_softmax, const T* input, T* output) { +__global__ void MaskedSoftmaxKernel(const int all_sequence_length, const int sequence_length, + const int* mask_end, const int* mask_start, + const T* add_before_softmax, const T* input, T* output) { __shared__ int start_position; __shared__ int end_position; @@ -338,42 +364,72 @@ __global__ void MaskedSoftmaxKernel(const int all_sequence_length, const int seq } __syncthreads(); - Softmax(all_sequence_length, sequence_length, end_position, start_position, add_before_softmax, input, output); + Softmax(all_sequence_length, sequence_length, end_position, start_position, + add_before_softmax, input, output); } template -__global__ void SoftmaxWithRawMaskSmallKernel(const int all_sequence_length, const int sequence_length, const int* attention_mask, const bool* key_padding_mask, const T* add_before_softmax, const T* input, - T* output, const bool is_unidirectional, const float rsqrt_head_size, const int mask_dimension, const int max_sequence_length, +__global__ void SoftmaxWithRawMaskSmallKernel(const int all_sequence_length, + const int sequence_length, + const int* attention_mask, + const bool* key_padding_mask, + const T* add_before_softmax, + const T* input, T* output, + const bool is_unidirectional, + const float rsqrt_head_size, + const int mask_dimension, + const int max_sequence_length, const bool skip_softmax) { - SoftmaxWithRawMaskSmall(all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, output, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, skip_softmax); + SoftmaxWithRawMaskSmall( + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, output, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + skip_softmax); } template -bool ComputeSoftmaxWithMask1D(hipStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, - const int* mask_index, const int* mask_start, const T* add_before_softmax, const T* input, T* output, const bool is_unidirectional) { +Status ComputeSoftmaxWithMask1D( + hipStream_t stream, + const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, + const int* mask_index, const int* mask_start, + const T* add_before_softmax, const T* input, T* output, const bool is_unidirectional) { const dim3 grid(sequence_length * num_heads, batch_size, 1); if (all_sequence_length <= 32) { const int blockSize = 32; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 64) { const int blockSize = 64; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 128) { const int blockSize = 128; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 256) { const int blockSize = 256; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 512) { const int blockSize = 512; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (all_sequence_length <= 1024) { const int blockSize = 1024; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output, is_unidirectional); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernelSmall), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output, is_unidirectional); } else if (!is_unidirectional) { const int blockSize = 1024; - hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, mask_index, mask_start, add_before_softmax, input, output); + hipLaunchKernelGGL(HIP_KERNEL_NAME(MaskedSoftmaxKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, mask_index, mask_start, + add_before_softmax, input, output); } else { ORT_THROW("Attention ROCM operator does not support total sequence length > 1024."); } @@ -382,42 +438,83 @@ bool ComputeSoftmaxWithMask1D(hipStream_t stream, const int all_sequence_length, } template -bool ComputeSoftmaxWithRawMask(hipStream_t stream, const int all_sequence_length, const int sequence_length, const int batch_size, const int num_heads, - const int* attention_mask, const bool* key_padding_mask, const T* add_before_softmax, const T* input, T* output, const bool is_unidirectional, - const float rsqrt_head_size, const int mask_dimension, const int max_sequence_length, const bool use_persistent_softmax, T* persistent_softmax_workspace) { +Status ComputeSoftmaxWithRawMask(hipStream_t stream, + const int all_sequence_length, + const int sequence_length, + const int batch_size, + const int num_heads, + const int* attention_mask, + const bool* key_padding_mask, + const T* add_before_softmax, + const T* input, + T* output, + const bool is_unidirectional, + const float rsqrt_head_size, + const int mask_dimension, + const int max_sequence_length, + const bool use_persistent_softmax, + T* persistent_softmax_workspace) { const dim3 grid(sequence_length * num_heads, batch_size, 1); T* out = use_persistent_softmax ? persistent_softmax_workspace : output; if (all_sequence_length <= 32) { const int blockSize = 32; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 64) { const int blockSize = 64; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 128) { const int blockSize = 128; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 256) { const int blockSize = 256; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 512) { const int blockSize = 512; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else if (all_sequence_length <= 1024) { const int blockSize = 1024; - hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, all_sequence_length, sequence_length, attention_mask, key_padding_mask, add_before_softmax, input, out, is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, use_persistent_softmax); + hipLaunchKernelGGL(HIP_KERNEL_NAME(SoftmaxWithRawMaskSmallKernel), grid, blockSize, 0, stream, + all_sequence_length, sequence_length, + attention_mask, key_padding_mask, add_before_softmax, input, out, + is_unidirectional, rsqrt_head_size, mask_dimension, max_sequence_length, + use_persistent_softmax); } else { ORT_THROW("Attention ROCM operator does not support total sequence length > 1024."); } if (use_persistent_softmax) { - dispatch_warpwise_softmax_forward(stream, output, persistent_softmax_workspace, all_sequence_length, all_sequence_length, batch_size * num_heads * sequence_length); + dispatch_warpwise_softmax_forward(stream, + output, + persistent_softmax_workspace, + all_sequence_length, + all_sequence_length, + batch_size * num_heads * sequence_length); } return HIP_CALL(hipPeekAtLastError()); } - } // namespace rocm } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc b/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc index d1a97f0d85..57ffcb6b4b 100644 --- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc +++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu.cc @@ -54,18 +54,13 @@ Status FastGelu::ComputeInternal(OpKernelContext* context) const { int64_t bias_length = (nullptr == bias) ? 0 : bias->Shape().Size(); typedef typename ToHipType::MappedType HipT; - if (!LaunchFastGeluKernel(Stream(), + return LaunchFastGeluKernel(Stream(), static_cast(input_length), static_cast(bias_length), reinterpret_cast(input->Data()), (nullptr != bias) ? reinterpret_cast(bias->Data()) : nullptr, reinterpret_cast(output->MutableData()), - tuning_)) { - HIP_CALL(hipGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + tuning_); } } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu index f7f0f67394..fc8ff3dcc6 100644 --- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu +++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.cu @@ -40,24 +40,23 @@ namespace contrib { namespace rocm { template -bool LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, +Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, const T* input, const T* bias, T* output, bool tuning) { static FastGeluTunableOp op; if (tuning) { op.EnableTuning(); } FastGeluParams op_params(stream, input, bias, output, input_length, bias_length); - op.Run(&op_params); - return HIP_CALL(hipPeekAtLastError()); + return op(&op_params); } -template bool LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, +template Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, const float* input, const float* bias, float* output, bool tuning); -template bool LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, +template Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, const BFloat16* input, const BFloat16* bias, BFloat16* output, bool tuning); -template bool LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, +template Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, const half* input, const half* bias, half* output, bool tuning); } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h index 767b622ff9..63eb6b052f 100644 --- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h +++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl.h @@ -6,13 +6,14 @@ // Licensed under the MIT License. #pragma once +#include "core/common/common.h" namespace onnxruntime { namespace contrib { namespace rocm { template -bool LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, +Status LaunchFastGeluKernel(hipStream_t stream, int input_length, int bias_length, const T* input, const T* bias, T* output, bool use_half2); } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl_kernel.h b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl_kernel.h index 6e112d6f73..79d5eb0122 100644 --- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl_kernel.h +++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_impl_kernel.h @@ -5,8 +5,6 @@ #include "contrib_ops/rocm/bert/util.h" -using namespace onnxruntime::rocm; - namespace onnxruntime { namespace contrib { namespace rocm { @@ -45,7 +43,6 @@ __global__ void FastGeluKernelVec(int input_length, int bias_length, const T* in const int idx = (blockIdx.x * TPB + threadIdx.x) * ILP; if (idx < input_length) { - using VecT = AlignedVector; T input_v[ILP]; VecT* input_val = reinterpret_cast(&input_v); *input_val = *reinterpret_cast(&input[idx]); @@ -59,7 +56,7 @@ __global__ void FastGeluKernelVec(int input_length, int bias_length, const T* in #pragma unroll for (int i = 0; i < ILP; i++) { - const T x = (bias == nullptr) ? input_v[i] : input_v[i] + bias_v[i]; + const T x = (bias == nullptr) ? input_v[i] : (T)(input_v[i] + bias_v[i]); const T u = twoT * x * (c * x * x + b); const T emu = __expf(-u); const T cdf = a + a * (twoT/(oneT + emu) - oneT); diff --git a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_tunable_op.h b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_tunable_op.h index 8096dbcbd2..ff019e9c4d 100644 --- a/onnxruntime/contrib_ops/rocm/bert/fast_gelu_tunable_op.h +++ b/onnxruntime/contrib_ops/rocm/bert/fast_gelu_tunable_op.h @@ -14,12 +14,12 @@ namespace onnxruntime { namespace contrib { namespace rocm { -template +template struct FastGeluParams : OpParams { FastGeluParams(hipStream_t stream, const T* input, const T* bias, T* output, int input_length, int bias_length) : OpParams(stream), input(input), bias(bias), output(output), input_length(input_length), bias_length(bias_length) {} - std::string signature() const { + std::string Signature() const override { std::string sig = std::to_string(input_length) + "_" + std::to_string(bias_length); return sig; } @@ -32,42 +32,33 @@ struct FastGeluParams : OpParams { }; template -void LaunchFastGelu(hipStream_t stream, const T* input, const T* bias, T* output, int input_length, int bias_length) { +Status FastGeluOp(const FastGeluParams* params) { + // TODO(anyone): Add tail handling for FastGelu + TUNABLE_OP_RETURN_UNSUPPOTED_ARGUMENT_IF( + !((params->bias_length > 0 && params->bias_length % VecSize == 0 && params->input_length % VecSize == 0) || + (params->bias_length == 0 && params->input_length % VecSize == 0))); + hipLaunchKernelGGL((FastGeluKernelVec), - dim3(CeilingDivision(input_length, ThreadsPerBlock*VecSize)), - dim3(ThreadsPerBlock), - 0, stream, - input_length, bias_length, input, bias, output); + dim3(CeilingDivision(params->input_length, ThreadsPerBlock * VecSize)), + dim3(ThreadsPerBlock), + 0, params->stream, + params->input_length, params->bias_length, params->input, params->bias, params->output); + auto status = hipGetLastError(); + ORT_RETURN_IF(status != hipSuccess, ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, hipGetErrorName(status))); + return Status::OK(); } +#define ADD_OP(threads_per_block) \ + this->ops_.emplace_back(FastGeluOp); \ + this->ops_.emplace_back(FastGeluOp); \ + this->ops_.emplace_back(FastGeluOp); \ + this->ops_.emplace_back(FastGeluOp); \ + this->ops_.emplace_back(FastGeluOp); -template -class FastGeluOp : public Op { +template +class FastGeluTunableOp : public TunableOp> { public: - FastGeluOp() : Op() {} - - void Run(const OpParams* op_params) { - const FastGeluParams* fast_gelu_params = static_cast*>(op_params); - LaunchFastGelu(fast_gelu_params->stream, - fast_gelu_params->input, - fast_gelu_params->bias, - fast_gelu_params->output, - fast_gelu_params->input_length, - fast_gelu_params->bias_length); - } -}; - -#define ADD_OP(threads_per_block) \ - ops_.push_back(std::make_unique>()); \ - ops_.push_back(std::make_unique>()); \ - ops_.push_back(std::make_unique>()); \ - ops_.push_back(std::make_unique>()); \ - ops_.push_back(std::make_unique>()); - -template -class FastGeluTunableOp : public TunableOp { - public: - FastGeluTunableOp() : TunableOp(15) { + FastGeluTunableOp() { ADD_OP(64); ADD_OP(128); ADD_OP(192); @@ -76,16 +67,14 @@ class FastGeluTunableOp : public TunableOp { ADD_OP(384); ADD_OP(448); ADD_OP(512); - } - private: - virtual bool Condition(const OpParams* op_params) { - const FastGeluParams* fast_gelu_params = static_cast*>(op_params); - bool condition = (fast_gelu_params->bias_length > 0) && (fast_gelu_params->bias_length % 16 == 0); - return condition; + // NOTE: the 15-th kernel seems to be better in gerenal case, so set it as default one + this->SetDefaultId(15); } }; +#undef ADD_OP + } // namespace rocm } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu.cc b/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu.cc index 427e005842..d19fd2377b 100644 --- a/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu.cc +++ b/onnxruntime/contrib_ops/rocm/bert/gemm_fast_gelu.cc @@ -69,17 +69,13 @@ Status GemmFastGelu::ComputeInternal(OpKernelContext* ctx) const { int64_t fast_gelu_input_length = Y->Shape().Size(); int64_t bias_length = (nullptr == bias) ? 0 : bias->Shape().Size(); - if (!LaunchFastGeluKernel(Stream(), + return LaunchFastGeluKernel(Stream(), static_cast(fast_gelu_input_length), static_cast(bias_length), reinterpret_cast(gemm_buffer.get()), (nullptr != bias) ? reinterpret_cast(bias->Data()) : nullptr, reinterpret_cast(Y->MutableData()), - false)) { - HIP_CALL(hipGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - return Status::OK(); + false); } } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc index 664bef1c24..4807c34355 100644 --- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc +++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm.cc @@ -96,7 +96,7 @@ Status SkipLayerNorm::ComputeInternal(OpKernelContext* ctx) const { size_t element_size = sizeof(T); typedef typename ToHipType::MappedType HipT; - if (!LaunchSkipLayerNormKernel( + return LaunchSkipLayerNormKernel( Stream(), reinterpret_cast(output->MutableData()), reinterpret_cast(input->Data()), @@ -107,13 +107,7 @@ Status SkipLayerNorm::ComputeInternal(OpKernelContext* ctx) const { epsilon_, hidden_size, static_cast(element_count), - element_size)) { - // Get last error to reset it to hipSuccess. - HIP_CALL(hipGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } - - return Status::OK(); + element_size); } } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu index 4eea85e11d..1b0b0cf69a 100644 --- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu +++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.cu @@ -112,7 +112,7 @@ __global__ void SkipLayerNormKernelSmall( } template -bool LaunchSkipLayerNormKernel( +Status LaunchSkipLayerNormKernel( hipStream_t stream, T* output, const T* input, const T* skip, const T* gamma, const T* beta, const T* bias, float epsilon, const int ld, const int element_count, size_t element_size) { @@ -178,15 +178,15 @@ bool LaunchSkipLayerNormKernel( return HIP_CALL(hipPeekAtLastError()); } -template bool LaunchSkipLayerNormKernel(hipStream_t stream, float* output, const float* input, - const float* skip, const float* gamma, const float* beta, - const float* bias, float epsilon, const int ld, - const int element_count, size_t element_size); +template Status LaunchSkipLayerNormKernel(hipStream_t stream, float* output, const float* input, + const float* skip, const float* gamma, const float* beta, + const float* bias, float epsilon, const int ld, + const int element_count, size_t element_size); -template bool LaunchSkipLayerNormKernel(hipStream_t stream, half* output, const half* input, - const half* skip, const half* gamma, const half* beta, - const half* bias, float epsilon, const int ld, - const int element_count, size_t element_size); +template Status LaunchSkipLayerNormKernel(hipStream_t stream, half* output, const half* input, + const half* skip, const half* gamma, const half* beta, + const half* bias, float epsilon, const int ld, + const int element_count, size_t element_size); } // namespace rocm } // namespace contrib diff --git a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h index 08ab0a5f06..d866a0bde6 100644 --- a/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h +++ b/onnxruntime/contrib_ops/rocm/bert/skip_layer_norm_impl.h @@ -2,13 +2,14 @@ // Licensed under the MIT License. #pragma once +#include "core/common/common.h" namespace onnxruntime { namespace contrib { namespace rocm { template -bool LaunchSkipLayerNormKernel( +Status LaunchSkipLayerNormKernel( hipStream_t stream, T* output, // output tensor const T* input, // input tensor @@ -25,4 +26,3 @@ bool LaunchSkipLayerNormKernel( } // namespace rocm } // namespace contrib } // namespace onnxruntime - diff --git a/onnxruntime/contrib_ops/rocm/bert/tunable_op.h b/onnxruntime/contrib_ops/rocm/bert/tunable_op.h index 7a28b76cc0..cebda8848c 100644 --- a/onnxruntime/contrib_ops/rocm/bert/tunable_op.h +++ b/onnxruntime/contrib_ops/rocm/bert/tunable_op.h @@ -3,11 +3,19 @@ #pragma once -#include +#include +#include + +#include +#include #include #include +#include +#include +#include #include -#include + +#include "core/common/common.h" #include "contrib_ops/rocm/bert/util.h" namespace onnxruntime { @@ -15,58 +23,70 @@ namespace contrib { namespace rocm { struct OpParams { + OpParams() : stream{} {} explicit OpParams(hipStream_t stream) : stream(stream) {} - virtual std::string signature() const = 0; + virtual std::string Signature() const = 0; hipStream_t stream; }; +// A type erased Callable wrapper. We could have used std::function> here. However, std::function +// requires the callable object to be CopyConstructible and CopyAssignable. This is not suitable for move only functor +// or move captured lambda. So we create a simple wrapper for our purpose here. +// +// Then an Op is Status(const ParamT*), that is, a callable accepts a const ParamT* and returns a Status. +// This means that it can be either a free function, a functor or a lambda. +template class Op { public: - Op() : repeats_(100) {} - - virtual void Run(const OpParams*) = 0; - - void SetRepeats(int n) { - repeats_ = n; - } - - float Profile(const OpParams* op_params) { - // warm up - for (int i = 0; i < 5; i++) { - Run(op_params); - } - timer_.Start(); - for (int i = 0; i < repeats_; i++) { - Run(op_params); - } - timer_.End(); - return timer_.time()/repeats_; - } - - virtual ~Op() {} + template + explicit Op(T&& c) : callable_{std::make_unique>(std::forward(c))} {} + Status operator()(const ParamT* param) { return (*callable_)(param); } private: - Timer timer_; - int repeats_; + struct ICallbale { + virtual ~ICallbale() = default; + virtual Status operator()(const ParamT*) = 0; + }; + + template + struct CallableImpl : ICallbale { + explicit CallableImpl(T&& c) : c_{std::move(c)} {} + Status operator()(const ParamT* param) override { return c_(param); } + + private: + T c_; + }; + + std::unique_ptr callable_; }; +// NOTE: onnxruntime's Status currently does not have a StatusCode::UNSUPPORTED. Currently, we do not want to extend the +// enum. So we reuse StatusCode::INVALID_ARGUMENT for this purpose. It can be interpreted as "The input argument is not +// valid for this specialized kernel implementation.". This semantic is crucial for the tuning mechanism. +#define TUNABLE_OP_RETURN_UNSUPPOTED_ARGUMENT_IF(condition, ...) \ + do { \ + if (condition) { \ + return ORT_MAKE_STATUS(NONE, INVALID_ARGUMENT, __VA_ARGS__); \ + } \ + } while (false) + +template class TunableOp { public: - explicit TunableOp(int default_id) : default_id_(default_id), tuning_(false) {} - - void Run(const OpParams* op_params) { + Status operator()(const ParamsT* params) { int id; - if (tuning_ == true && Condition(op_params)) { - if (kernel_map_.find(op_params->signature()) == kernel_map_.end()) { - id = FindFastest(op_params); - kernel_map_.insert({op_params->signature(), id}); + if (tuning_) { + if (kernel_map_.find(params->Signature()) == kernel_map_.end()) { + id = FindFastest(params); + kernel_map_.insert({params->Signature(), id}); } else { - id = kernel_map_[op_params->signature()]; + id = kernel_map_[params->Signature()]; } } else { id = default_id_; } - ops_[id]->Run(op_params); + ORT_RETURN_IF_ERROR(ops_[id](params)); + return Status::OK(); } void EnableTuning() { @@ -77,31 +97,73 @@ class TunableOp { tuning_ = false; } - virtual ~TunableOp() {} + virtual ~TunableOp() = default; protected: - std::vector> ops_; + // set the default op to be used in non-tuning scenario + void SetDefaultId(int id) { + ORT_ENFORCE(id < ops_.size(), "TunableOp id out of bound"); + default_id_ = id; + } private: - virtual bool Condition(const OpParams* op_params) = 0; + static void WarmUp(Op& op, const ParamsT* param) { + const int num_iter = 4; + for (int i = 0; i < num_iter; i++) { + ORT_THROW_IF_ERROR(op(param)); + } + } - int FindFastest(const OpParams* op_params) { - assert(ops_.size() > 0); - float min_time = ops_[0]->Profile(op_params); - int id = 0; - for (int i = 1; i < ops_.size(); i++) { - float time = ops_[i]->Profile(op_params); + static double Profile(Op& op, const ParamsT* param) { + const int num_iter = 100; + Timer timer{}; + timer.Start(); + for (int i = 0; i < num_iter; i++) { + ORT_THROW_IF_ERROR(op(param)); + } + timer.End(); + return timer.Duration() / num_iter; + } + + static bool IsSupported(Op& op, const ParamsT* param) { + Status status = op(param); + if (status.Category() == common::StatusCategory::NONE && status.Code() == common::StatusCode::INVALID_ARGUMENT) { + return false; + } + ORT_THROW_IF_ERROR(status); + return true; + } + + int FindFastest(const ParamsT* params) { + auto min_time = std::numeric_limits::infinity(); + int id = -1; + for (size_t i = 0; i < this->ops_.size(); i++) { + if (!IsSupported(ops_[i], params)) { + continue; + } + + WarmUp(ops_[i], params); + auto time = Profile(ops_[i], params); if (time < min_time) { min_time = time; - id = i; + id = static_cast(i); } } + ORT_ENFORCE(id >= 0, "Cannot found viable op"); return id; } - std::map kernel_map_; - int default_id_; - bool tuning_; + protected: + std::vector> ops_; + + private: + // mapping from Signature to best impl + std::unordered_map kernel_map_; + + // the default impl to use when tuning is disabled + int default_id_{0}; + + bool tuning_{false}; }; } // namespace rocm diff --git a/onnxruntime/contrib_ops/rocm/bert/util.cc b/onnxruntime/contrib_ops/rocm/bert/util.cc index a5bc0929f8..2ed0a8d162 100644 --- a/onnxruntime/contrib_ops/rocm/bert/util.cc +++ b/onnxruntime/contrib_ops/rocm/bert/util.cc @@ -27,7 +27,7 @@ void Timer::End() { HIP_CHECK(hipEventSynchronize(end_)); } -float Timer::time() { +float Timer::Duration() { float time; // time is in ms with a resolution of 1 us HIP_CHECK(hipEventElapsedTime(&time, start_, end_)); diff --git a/onnxruntime/contrib_ops/rocm/bert/util.h b/onnxruntime/contrib_ops/rocm/bert/util.h index d69754e677..9c5c2900e0 100644 --- a/onnxruntime/contrib_ops/rocm/bert/util.h +++ b/onnxruntime/contrib_ops/rocm/bert/util.h @@ -32,7 +32,7 @@ class Timer { Timer(); void Start(); void End(); - float time(); + float Duration(); ~Timer(); private: diff --git a/onnxruntime/core/codegen/common/utils.cc b/onnxruntime/core/codegen/common/utils.cc index 9912df1868..f4140a411b 100644 --- a/onnxruntime/core/codegen/common/utils.cc +++ b/onnxruntime/core/codegen/common/utils.cc @@ -70,9 +70,6 @@ TargetFeature GetTargetInfo(const codegen::CodeGenSettings& settings) { TargetFeature feature; std::string target_str = ""; - if (settings.HasOption(nuphar::kNupharCodeGenTarget)) { - target_str = settings.GetOptionValue(nuphar::kNupharCodeGenTarget); - } bool isAVX = false; bool isAVX2 = false; diff --git a/onnxruntime/core/codegen/common/utils.h b/onnxruntime/core/codegen/common/utils.h index 4e3e5c1fed..ef06b5b72d 100644 --- a/onnxruntime/core/codegen/common/utils.h +++ b/onnxruntime/core/codegen/common/utils.h @@ -3,7 +3,6 @@ #pragma once #include "core/common/common.h" -#include "core/providers/nuphar/common/nuphar_settings.h" #include #include #include diff --git a/onnxruntime/core/common/profiler_common.cc b/onnxruntime/core/common/profiler_common.cc new file mode 100644 index 0000000000..ee14533b2a --- /dev/null +++ b/onnxruntime/core/common/profiler_common.cc @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/profiler_common.h" + +#ifndef _MSC_VER +#include +#endif +#include +#include + +namespace onnxruntime { +namespace profiling { + +static constexpr int kMaxSymbolSize = 1024; + +std::string demangle(const char* name) { +#ifndef _MSC_VER + if (!name) { + return ""; + } + + if (strlen(name) > kMaxSymbolSize) { + return name; + } + + int status; + size_t len = 0; + char* demangled = abi::__cxa_demangle(name, nullptr, &len, &status); + if (status != 0) { + return name; + } + std::string res(demangled); + // The returned buffer must be freed! + free(demangled); + return res; +#else + // TODO(anyone): demangling on Windows + if (!name) { + return ""; + } else { + return name; + } +#endif +} + +std::string demangle(const std::string& name) { + return demangle(name.c_str()); +} + +} // namespace profiling +} // namespace onnxruntime + diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index 0dcae9a1c0..7d3a6e06a6 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -825,9 +825,9 @@ void ExecutionFrame::VerifyOutputSizes(int output_index, const Node& node, const } if (!compatible) { - LOGS(session_state_.Logger(), WARNING) << "Expected shape from model of " << *expected_shape - << " does not match actual shape of " << output_shape - << " for output " << output_def->Name(); + LOGS(session_state_.Logger(), WARNING) + << "Expected shape from model of " << utils::GetTensorShapeFromTensorShapeProto(*expected_shape) + << " does not match actual shape of " << output_shape << " for output " << output_def->Name(); } } diff --git a/onnxruntime/core/framework/graph_partitioner.cc b/onnxruntime/core/framework/graph_partitioner.cc index 74854e8bec..79f3a389b8 100644 --- a/onnxruntime/core/framework/graph_partitioner.cc +++ b/onnxruntime/core/framework/graph_partitioner.cc @@ -251,11 +251,8 @@ static Node* PlaceNode(Graph& graph, const IndexedSubGraph& capability, // Ideally, it should be moved to "Compile" call. // Here we temporary keep the function body for DML fusion // Need to remove it after migrate DML to the Compile-based approach. - // TODO2: Nuphar is out of maintain, keep it with old API temporarily. - // We want to remove Nuphar soon. if (fusion_style == IExecutionProvider::FusionStyle::Function || - provider_type == kDmlExecutionProvider || - provider_type == kNupharExecutionProvider) { + provider_type == kDmlExecutionProvider) { fused_node = &graph.FuseSubGraph(capability, node_name); } else { // create a fused node without copying everything to a Function body. The IndexedSubGraph will be passed @@ -373,8 +370,6 @@ static Status PartitionOnnxFormatModelImpl(Graph& graph, FuncManager& func_mgr, std::vector node_compute_funcs; // !!! The Function style fusion is deprecated. if (fusion_style == IExecutionProvider::FusionStyle::Function) { - // TODO: Nuphar is out of maintain. Use the old api temporarily. - // We want to remove it soon. // Create a Function based node where the fused nodes have a new Graph instance. static std::once_flag legacy_compile_method_warning_flag; std::call_once( diff --git a/onnxruntime/core/framework/kernel_def_builder.cc b/onnxruntime/core/framework/kernel_def_builder.cc index 3c773248e0..b3fffe602f 100644 --- a/onnxruntime/core/framework/kernel_def_builder.cc +++ b/onnxruntime/core/framework/kernel_def_builder.cc @@ -45,7 +45,16 @@ void KernelDef::CalculateHash() { // kernel definition) hash_str(op_name_); - hash_int(op_since_version_start_); + + if constexpr (onnxruntime::endian::native == onnxruntime::endian::little) { + hash_int(op_since_version_start_); + } else { + const uint8_t* c = (const uint8_t*)&op_since_version_start_; + hash_int((uint32_t)c[0] | + (uint32_t)c[1] << 8 | + (uint32_t)c[2] << 16 | + (uint32_t)c[3] << 24); + } // If we include op_since_version_end_ the hash of an existing op changes when it's superseded. // e.g. Unsqueeze 11 had no end version until Unsqueeze 13, at which point the existing op is changed to have diff --git a/onnxruntime/core/framework/ort_value_name_idx_map.h b/onnxruntime/core/framework/ort_value_name_idx_map.h index 6035dc4e85..1b5f6bcee9 100644 --- a/onnxruntime/core/framework/ort_value_name_idx_map.h +++ b/onnxruntime/core/framework/ort_value_name_idx_map.h @@ -33,7 +33,11 @@ class OrtValueNameIdxMap { common::Status GetIdx(std::string_view name, int& idx) const { idx = -1; +#ifdef DISABLE_ABSEIL + auto it = map_.find(std::string(name)); +#else auto it = map_.find(name); +#endif if (it == map_.end()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Could not find OrtValue with name '", name, "'"); } diff --git a/onnxruntime/core/framework/session_options.h b/onnxruntime/core/framework/session_options.h index c3012299a3..827cb3c257 100644 --- a/onnxruntime/core/framework/session_options.h +++ b/onnxruntime/core/framework/session_options.h @@ -83,7 +83,7 @@ struct SessionOptions { std::string session_logid; ///< logger id to use for session output /// Log severity for the inference session. Applies to session load, initialization, etc. - /// See https://github.com/microsoft/onnxruntime/blob/master/include/onnxruntime/core/common/logging/severity.h + /// See https://github.com/microsoft/onnxruntime/blob/main/include/onnxruntime/core/common/logging/severity.h /// Default = -1 (use default logger severity) int session_log_severity_level = -1; int session_log_verbosity_level = 0; ///< VLOG level if debug build and session_log_severity_level is 0 (VERBOSE). diff --git a/onnxruntime/core/framework/session_state_utils.cc b/onnxruntime/core/framework/session_state_utils.cc index a322ae2354..0d0015e9bd 100644 --- a/onnxruntime/core/framework/session_state_utils.cc +++ b/onnxruntime/core/framework/session_state_utils.cc @@ -87,9 +87,7 @@ static inline common::Status ExtDataTensorProtoToTensor(const Env& env, // avoided if the Tensor class implements the do-nothing behavior when given a // nullptr for the allocator argument const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); - std::vector tensor_shape_vec = utils::GetTensorShapeFromTensorProto(tensor_proto); - TensorShape tensor_shape{tensor_shape_vec}; - + TensorShape tensor_shape = utils::GetTensorShapeFromTensorProto(tensor_proto); tensor = Tensor(type, tensor_shape, ext_data_buf, OrtMemoryInfo(CPU, OrtAllocatorType::OrtDeviceAllocator)); return common::Status::OK(); @@ -106,7 +104,7 @@ static common::Status DeserializeTensorProto(const Env& env, const std::basic_st } // Get shape and type of the tensor, and allocate the empty tensor - TensorShape tensor_shape{utils::GetTensorShapeFromTensorProto(tensor_proto)}; + TensorShape tensor_shape = utils::GetTensorShapeFromTensorProto(tensor_proto); const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); std::unique_ptr p_tensor; if (m != nullptr) { diff --git a/onnxruntime/core/framework/tensorprotoutils.cc b/onnxruntime/core/framework/tensorprotoutils.cc index ce384545b8..02a5dcb707 100644 --- a/onnxruntime/core/framework/tensorprotoutils.cc +++ b/onnxruntime/core/framework/tensorprotoutils.cc @@ -494,14 +494,14 @@ TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShape return TensorShape(std::move(tensor_shape_vec)); } -std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto) { +TensorShape GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto) { const auto& dims = tensor_proto.dims(); std::vector tensor_shape_vec(static_cast(dims.size())); for (int i = 0; i < dims.size(); ++i) { tensor_shape_vec[i] = dims[i]; } - return tensor_shape_vec; + return TensorShape(std::move(tensor_shape_vec)); } struct UnInitializeParam { @@ -650,8 +650,8 @@ Status TensorProtoToTensor(const Env& env, const ORTCHAR_T* model_path, const ONNX_NAMESPACE::TensorProto& tensor_proto, Tensor& tensor) { // Validate tensor compatibility - std::vector tensor_shape_vec = GetTensorShapeFromTensorProto(tensor_proto); - if (gsl::make_span(tensor_shape_vec) != tensor.Shape().GetDims()) { + TensorShape tensor_shape = GetTensorShapeFromTensorProto(tensor_proto); + if (tensor_shape != tensor.Shape()) { return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "TensorProtoToTensor() tensor shape mismatch!"); } const DataTypeImpl* const source_type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); @@ -747,7 +747,7 @@ Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* model_path, } // Note: We permit an empty tensor_shape_vec, and treat it as a scalar (a tensor of size 1). - TensorShape tensor_shape{GetTensorShapeFromTensorProto(tensor_proto)}; + TensorShape tensor_shape = GetTensorShapeFromTensorProto(tensor_proto); const DataTypeImpl* const type = DataTypeImpl::TensorTypeFromONNXEnum(tensor_proto.data_type())->GetElementType(); std::unique_ptr tensorp = std::make_unique(type, tensor_shape, m.GetBuffer(), m.GetAllocInfo()); if (tensorp->SizeInBytes() > m.GetLen()) { @@ -798,7 +798,7 @@ ONNXTensorElementDataType GetTensorElementType(const ONNX_NAMESPACE::TensorProto ONNX_NAMESPACE::TensorProto TensorToTensorProto(const Tensor& tensor, const std::string& tensor_proto_name) { // Given we are using the raw_data field in the protobuf, this will work only for little-endian format. - if constexpr(endian::native != endian::little) { + if constexpr (endian::native != endian::little) { ORT_THROW("Big endian not supported"); } @@ -1146,10 +1146,9 @@ static void SetIndices(gsl::span gathered_indices, auto* ind_dest = reinterpret_cast(raw_indices.data()); size_t dest_index = 0; for (auto src_index : gathered_indices) { - if constexpr(sizeof(T) == sizeof(int8_t)) { + if constexpr (sizeof(T) == sizeof(int8_t)) { ind_dest[dest_index] = static_cast(src_index); - } - else { + } else { auto* dst = ind_dest + dest_index; T v = static_cast(src_index); memcpy(dst, &v, sizeof(T)); diff --git a/onnxruntime/core/framework/tensorprotoutils.h b/onnxruntime/core/framework/tensorprotoutils.h index 1edfcedc3d..6f1b60dd29 100644 --- a/onnxruntime/core/framework/tensorprotoutils.h +++ b/onnxruntime/core/framework/tensorprotoutils.h @@ -36,7 +36,7 @@ namespace utils { TensorShape GetTensorShapeFromTensorShapeProto(const ONNX_NAMESPACE::TensorShapeProto& tensor_shape_proto); -std::vector GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto); +TensorShape GetTensorShapeFromTensorProto(const ONNX_NAMESPACE::TensorProto& tensor_proto); /** * deserialize a TensorProto into a preallocated memory buffer. diff --git a/onnxruntime/core/framework/utils.cc b/onnxruntime/core/framework/utils.cc index 8059de6c84..9199c41f35 100644 --- a/onnxruntime/core/framework/utils.cc +++ b/onnxruntime/core/framework/utils.cc @@ -27,50 +27,6 @@ #include "contrib_ops/cpu/aten_ops/aten_op_executor.h" #endif -namespace ONNX_NAMESPACE { -std::ostream& operator<<(std::ostream& out, const TensorShapeProto& shape_proto) { - std::string result; - result.reserve(128); - - result.append("{"); - bool first = true; - for (auto& dim : shape_proto.dim()) { - if (!first) { - result.append(","); - } - - if (onnxruntime::utils::HasDimValue(dim)) - result.append(std::to_string(dim.dim_value())); - else if (onnxruntime::utils::HasDimParam(dim)) - result.append(dim.dim_param()); - - first = false; - } - result.append("}"); - - return (out << result); -} - -std::ostream& operator<<(std::ostream& out, const TensorProto& tensor_proto) { - std::string result; - result.reserve(128); - - result.append("{"); - bool first = true; - for (auto& dim : tensor_proto.dims()) { - if (!first) { - result.append(","); - } - - result.append(std::to_string(dim)); - first = false; - } - result.append("}"); - - return (out << result); -} -} // namespace ONNX_NAMESPACE - namespace onnxruntime { namespace utils { void* DefaultAlloc(size_t size) { @@ -98,7 +54,6 @@ void DestroyStrings(void* p_data, int64_t elements) { bool ProviderIsCpuBased(const std::string& provider_type) { return provider_type == onnxruntime::kCpuExecutionProvider || provider_type == onnxruntime::kDnnlExecutionProvider || - provider_type == onnxruntime::kNupharExecutionProvider || provider_type == onnxruntime::kTvmExecutionProvider || provider_type == onnxruntime::kVitisAIExecutionProvider || provider_type == onnxruntime::kOpenVINOExecutionProvider || diff --git a/onnxruntime/core/framework/utils.h b/onnxruntime/core/framework/utils.h index 84c96757bb..ebe599193d 100644 --- a/onnxruntime/core/framework/utils.h +++ b/onnxruntime/core/framework/utils.h @@ -13,12 +13,6 @@ #ifdef ENABLE_TRAINING #include "core/framework/partial_graph_execution_state.h" #endif -namespace ONNX_NAMESPACE { -class TensorShapeProto; -class TensorProto; -std::ostream& operator<<(std::ostream& out, const TensorShapeProto& shape_proto); -std::ostream& operator<<(std::ostream& out, const TensorProto& tensor_proto); -} // namespace ONNX_NAMESPACE namespace onnxruntime { class ExecutionProviders; diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index d1f051f46f..921d44716f 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -1295,7 +1295,8 @@ activation and leaky_relu_alpha.)DOC") "C", "Input tensor C. " "The shape of C should be unidirectional broadcastable to (M, N).", - "T") + "T", + OpSchema::Optional) .Output(0, "Y", "Output tensor of shape (M, N).", "T") .TypeConstraint( "T", diff --git a/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc b/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc index b45757efb9..9b043a4ea7 100644 --- a/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc +++ b/onnxruntime/core/graph/contrib_ops/internal_nhwc_onnx_opset.cc @@ -80,12 +80,15 @@ void OpSet_Internal_NHWC_ONNX::ForEachSchema(const std::function()); fn(GetOpSchema()); fn(GetOpSchema()); + fn(GetOpSchema()); + fn(GetOpSchema()); + fn(GetOpSchema()); fn(GetOpSchema()); fn(GetOpSchema()); diff --git a/onnxruntime/core/graph/contrib_ops/quantization_defs.cc b/onnxruntime/core/graph/contrib_ops/quantization_defs.cc index 34fb5a16ad..ccc3dd3b93 100644 --- a/onnxruntime/core/graph/contrib_ops/quantization_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/quantization_defs.cc @@ -1131,5 +1131,89 @@ If mask is provided, mask index (that is position of first 0 in mask, or number .TypeConstraint("T2", {"tensor(int8)", "tensor(uint8)"}, "Constrain input and output types to int8 tensors.") .TypeConstraint("T", {"tensor(float)"}, "Constrain input and output types to float32 tensors.") .TypeAndShapeInferenceFunction(EmbedLayerNormalizationShapeInference)); + + constexpr const char* QOrderedMatMul_ver1_doc = R"DOC(TODO)DOC"; + + ONNX_MS_OPERATOR_SET_SCHEMA(QOrderedMatMul, 1, + OpSchema() + .SetDoc(QOrderedMatMul_ver1_doc) + .Attr("order_A", "cublasLt order of matrix A. Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .Attr("order_B", "cublasLt order of matrix B. Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .Attr("order_Y", + "cublasLt order of matrix Y and optional matrix C. Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .Input(0, "A", "3-dimensional matrix A", "Q") + .Input(1, "scale_A", "scale of the input A", "S") + .Input(2, "B", "2-dimensional matrix B", "Q") + .Input(3, "scale_B", "scale of the input B", "S") + .Input(4, "scale_Y", "scale of the output Y", "S") + .Input(5, "bias", "1d bias", "S", OpSchema::Optional) + .Input(6, "C", + "3d or 2d matrix C. if 2d expand to 3d first. " + "Shape[0] should be 1 or same as A.shape[0] ", + "Q", OpSchema::Optional) + .Input(7, "scale_C", "scale of the input A", "S", OpSchema::Optional) + .Output(0, "Y", "Matrix multiply results from A * B", "Q") + .TypeConstraint("Q", {"tensor(int8)"}, + "Constrain input and output types to int8 tensors.") + .TypeConstraint("S", {"tensor(float)"}, "Constrain bias and scales to float32") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromInputToOutput(ctx, 0, 0); + ONNX_NAMESPACE::matmulShapeInference(ctx, 0, 2); + })); + + ONNX_MS_OPERATOR_SET_SCHEMA(QOrderedLayerNormalization, 1, + OpSchema() + .SetDoc("QOrderedLayerNormalization") + .Attr("axis", + "The first normalization dimension: normalization " + "will be performed along dimensions axis " + ": rank(inputs).", + AttributeProto::INT, + static_cast(-1)) + .Attr("epsilon", "The epsilon value to use to avoid division by zero.", + AttributeProto::FLOAT, 1e-5f) + .Attr("order_X", "cublasLt order of input X. Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .Attr("order_Y", + "cublasLt order of matrix Y, must be same as order_X. " + "Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .AllowUncheckedAttributes() + .Input(0, "X", "Input data tensor from the previous layer.", "Q") + .Input(1, "scale_X", "scale of the quantized X", "S") + .Input(2, "scale", "Scale tensor, i.e., gamma vector.", "F") + .Input(3, "B", "Bias tensor.", "F", OpSchema::Optional) + .Input(4, "scale_Y", "scale of the quantized X", "S") + .Output(0, "Y", "Output data tensor.", "Q") + .TypeConstraint("F", {"tensor(float16)", "tensor(float)"}, + "Constrain input gamma and bias could be float16/float tensors. " + "float may get better precision, float16 runs faster.") + .TypeConstraint("S", {"tensor(float)"}, "quantization scale must be float tensors.") + .TypeConstraint("Q", {"tensor(int8)"}, "quantization tensor must be int8 tensors.") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateShapeAndTypeFromFirstInput(ctx); + propagateElemTypeFromInputToOutput(ctx, 0, 0); + })); + + ONNX_MS_OPERATOR_SET_SCHEMA(QOrderedGelu, 1, + OpSchema() + .SetDoc(R"DOC(Ordered Quantize Gelu.)DOC") + .Attr("order_X", "cublasLt order of input X. Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .Attr("order_Y", + "cublasLt order of matrix Y, must be same as order_X. " + "Default is ROW MAJOR.", + AttributeProto::INT, static_cast(1)) + .Input(0, "X", "N-dimensional input A", "Q") + .Input(1, "scale_X", "scale of the input A", "S") + .Input(2, "scale_Y", "scale of the output Y", "S") + .Output(0, "Y", "Output of the Gelu", "Q") + .TypeConstraint("Q", {"tensor(int8)"}, + "Constrain input and output types to int8 tensors.") + .TypeConstraint("S", {"tensor(float)"}, "Constrain scales to float32") + .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput)); } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/graph/function_utils.cc b/onnxruntime/core/graph/function_utils.cc index 1ea90439fc..eb7655cb65 100644 --- a/onnxruntime/core/graph/function_utils.cc +++ b/onnxruntime/core/graph/function_utils.cc @@ -26,12 +26,13 @@ static int GetVersionForDomain(const std::string& domain, const Msecond; } -std::unique_ptr -CreateSchema(const Graph& graph, - const IndexedSubGraph& nodes_to_fuse) { +std::unique_ptr CreateSchema( + const Graph& graph, + const IndexedSubGraph& nodes_to_fuse, bool allow_aggregated_tensor_type) { const auto* meta_def = nodes_to_fuse.GetMetaDef(); - auto op_schema = std::make_unique(); - op_schema->SetName(meta_def->name); + + using ONNX_NAMESPACE::OpSchema; + auto op_schema = std::make_unique(meta_def->name, __FILE__, __LINE__); op_schema->SetDomain(meta_def->domain); op_schema->SetDoc(meta_def->doc_string); op_schema->SinceVersion(meta_def->since_version); @@ -40,20 +41,34 @@ CreateSchema(const Graph& graph, op_schema->TypeAndShapeInferenceFunction(meta_def->type_and_shape_inference_function); } - int i = 0; + if (allow_aggregated_tensor_type) { + // The generated schema will use the same type constraint for all inputs and outputs, + // and that type constraint will match all tensor types. + // Due to this, a user of this style of schema must manually check whether any applicable type constraints + // for each input or output are satisfied prior to creating a node that uses this schema + // + op_schema->TypeConstraint("TAggregatedTypes", ONNX_NAMESPACE::OpSchema::all_tensor_types_with_bfloat(), + "all_tensor_types_with_bfloat"); + } - for (auto& input : meta_def->inputs) { - auto input_arg = graph.GetNodeArg(input); + int i = 0; + for (const auto& input : meta_def->inputs) { + const auto *input_arg = graph.GetNodeArg(input); // inputs must have a type. can be inferred for outputs. ORT_ENFORCE(input_arg->Type() != nullptr); - op_schema->Input(i, input, "", *input_arg->Type()); - ++i; + op_schema->Input(i, input, "", + allow_aggregated_tensor_type ? "TAggregatedTypes" : *input_arg->Type(), + OpSchema::FormalParameterOption::Single, /*is_homogeneous=*/!allow_aggregated_tensor_type); + i++; } + i = 0; - for (auto& output : meta_def->outputs) { - auto output_arg = graph.GetNodeArg(output); - op_schema->Output(i, output, "", *output_arg->Type()); - ++i; + for (const auto& output : meta_def->outputs) { + const auto* output_arg = graph.GetNodeArg(output); + op_schema->Output(i, output, "", + allow_aggregated_tensor_type ? "TAggregatedTypes" : *output_arg->Type(), + OpSchema::FormalParameterOption::Single, /*is_homogeneous=*/!allow_aggregated_tensor_type); + i++; } op_schema->Finalize(); @@ -278,7 +293,7 @@ std::unique_ptr CreateSchema(const std::string& functi } class Inliner { -private: + private: std::string prefix; const onnxruntime::NodeAttributes& attr_map; std::vector> rename_scopes; @@ -359,7 +374,10 @@ private: // if the call-node contains the attribute. Otherwise, this attribute must be removed. auto entry = attr_map.find(attr.ref_attr_name()); if (entry != attr_map.cend()) { + // Copy value of attribute, but retain original name: + std::string name = attr.name(); attr = entry->second; + attr.set_name(name); } else { attr_iter = attributes.erase(attr_iter); continue; @@ -369,7 +387,7 @@ private: if (attr.has_g()) { transform(*attr.mutable_g()); } - for (auto& graph: *attr.mutable_graphs()) + for (auto& graph : *attr.mutable_graphs()) transform(graph); ++attr_iter; } @@ -380,6 +398,8 @@ private: rename_scopes.emplace_back(); for (auto& x : *graph.mutable_input()) make_unique(*x.mutable_name()); + for (auto& init : *graph.mutable_initializer()) + make_unique(*init.mutable_name()); for (auto& y : *graph.mutable_output()) make_unique(*y.mutable_name()); for (auto& n : *graph.mutable_node()) @@ -387,7 +407,7 @@ private: rename_scopes.pop_back(); } -public: + public: // The main specialization method: specialize a FunctionProto for a particular call-site. static void specialize(const NodeProto& callnode, FunctionProto& callee, const onnxruntime::NodeAttributes& attr_map, std::string unique_prefix) { Inliner inliner(unique_prefix, attr_map); diff --git a/onnxruntime/core/graph/function_utils.h b/onnxruntime/core/graph/function_utils.h index 59c665d92f..fb67bc2948 100644 --- a/onnxruntime/core/graph/function_utils.h +++ b/onnxruntime/core/graph/function_utils.h @@ -14,13 +14,17 @@ namespace onnxruntime { namespace function_utils { -/** Create a OpSchema given a subgraph EP whant to fuse. -* This is used when EP return fusion in GetCapability implementation. -* @param graph The graph which host the subgraph. -* @param nodes_to_fuse The metadata for the subgraph that EP want to fuse. -*/ +/** Create a OpSchema given a subgraph EP want to fuse. + * This is used when EP return fusion in GetCapability implementation. + * @param graph The graph which host the subgraph. + * @param nodes_to_fuse The metadata for the subgraph that EP want to fuse. + * @param allow_aggregated_tensor_type if true, it will use a type constraint called + * TAggregatedTypes for all inputs and outputs, + * and that this will match all tensor types in the all_tensor_types_with_bfloat list. + */ std::unique_ptr CreateSchema(const Graph& graph, - const IndexedSubGraph& nodes_to_fuse); + const IndexedSubGraph& nodes_to_fuse, + bool allow_aggregated_tensor_type = false); /** Create a OpSchema given from a local function in onnx model. * @param function_domain The domain of the function. diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 03078dca09..089aa6cd91 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -10,6 +10,7 @@ #include #include +#include "core/common/common.h" #include "gsl/gsl" #include "core/common/logging/logging.h" #include "core/common/inlined_containers.h" @@ -104,7 +105,8 @@ static Status MergeShapeInfo(const std::string& output_name, // target.shape() was empty. 'assert' just in case that ever changes. assert(utils::HasShape(source) && utils::HasShape(target)); LOGS(logger, WARNING) << "Error merging shape info for output. '" << output_name - << "' source:" << utils::GetShape(source) << " target:" << utils::GetShape(target) + << "' source:" << utils::GetTensorShapeFromTensorShapeProto(utils::GetShape(source)) + << " target:" << utils::GetTensorShapeFromTensorShapeProto(utils::GetShape(target)) << ". Falling back to lenient merge."; if (utils::HasTensorType(source)) { ONNX_NAMESPACE::UnionShapeInfo(utils::GetShape(source), *target.mutable_tensor_type()); @@ -167,6 +169,12 @@ static TypeProto TypeProtoFromTensorProto(const TensorProto& tensor) { return t; } + +static std::string GenerateSchemaKey(const IndexedSubGraph& subgraph_ptr) { + return MakeString(subgraph_ptr.GetMetaDef()->domain, "_", + subgraph_ptr.GetMetaDef()->name, "_", + subgraph_ptr.GetMetaDef()->since_version); +} #endif // !defined(ORT_MINIMAL_BUILD) #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) @@ -2413,19 +2421,25 @@ common::Status Graph::TypeCheckInputsAndInitializers() { node_arg->SetShape(inferred_shape); } } else { + bool invalid = false; + if (p_existing_shape->dim_size() != tensor_proto->dims_size()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Type Error: Shape of initializer ", name, " does not match. ", - *p_existing_shape, " != ", *tensor_proto); + invalid = true; + } else { + for (int i = 0; i < p_existing_shape->dim_size(); ++i) { + auto& d = p_existing_shape->dim(i); + if (utils::HasDimValue(d) && (d.dim_value() != tensor_proto->dims(i))) { + invalid = true; + break; + } + } } - for (int i = 0; i < p_existing_shape->dim_size(); ++i) { - auto& d = p_existing_shape->dim(i); - if (utils::HasDimValue(d) && (d.dim_value() != tensor_proto->dims(i))) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Type Error: Shape of initializer ", name, " does not match. ", - *p_existing_shape, " != ", *tensor_proto); - } + if (invalid) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Type Error: Shape of initializer ", name, " does not match. ", + utils::GetTensorShapeFromTensorShapeProto(*p_existing_shape), " != ", + utils::GetTensorShapeFromTensorProto(*tensor_proto)); } } } @@ -2554,6 +2568,15 @@ Status Graph::VerifyNodeAndOpMatch(const ResolveOptions& options) { } } + // verify subgraphs + for (auto node_index : nodes_in_topological_order_) { + auto& node = *GetNode(node_index); + for (auto& entry : node.GetAttributeNameToMutableSubgraphMap()) { + Graph* subgraph = entry.second; + ORT_RETURN_IF_ERROR(subgraph->VerifyNodeAndOpMatch(options)); + } + } + return Status::OK(); } @@ -3788,13 +3811,13 @@ Node& Graph::CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std std::unordered_map output_indexes; int cur_idx = 0; - for (auto& arg_name : func_meta_def->inputs) { + for (const auto& arg_name : func_meta_def->inputs) { input_args.push_back(GetNodeArg(arg_name)); input_indexes[arg_name] = cur_idx++; } cur_idx = 0; - for (auto& arg_name : func_meta_def->outputs) { + for (const auto& arg_name : func_meta_def->outputs) { output_args.push_back(GetNodeArg(arg_name)); output_indexes[arg_name] = cur_idx++; } @@ -3813,12 +3836,20 @@ Node& Graph::CreateFusedSubGraphNode(const IndexedSubGraph& sub_graph, const std // kernel lookup works as per usual, if not using an existing schema. // in an extended minimal build we do the lookup via a hash so don't need a schema. fused_node.SetSinceVersion(func_meta_def->since_version); - if (sub_graph.use_existing_schema) { + if (sub_graph.schema_source == IndexedSubGraph::SourceOfSchema::EXISTING) { ORT_ENFORCE(SetOpSchemaFromRegistryForNode(fused_node), "Schema was not found for fused node. Domain:", fused_node.Domain(), " OpType:", fused_node.OpType()); + } else if (IndexedSubGraph::SourceOfSchema::REUSE_OR_CREATE == sub_graph.schema_source) { + auto schema_key = GenerateSchemaKey(sub_graph); + if (!reusable_fused_schema_map_.contains(schema_key)) { + fused_schemas_containers_.push_back( + function_utils::CreateSchema(*this, sub_graph, /*allow_aggregated_tensor_type=*/true)); + reusable_fused_schema_map_.emplace(schema_key, *fused_schemas_containers_.back()); + } + + fused_node.op_ = &(reusable_fused_schema_map_.at(schema_key).get()); } else { - auto temp_schema_ptr = function_utils::CreateSchema(*this, sub_graph); - fused_schemas_containers_.push_back(std::move(temp_schema_ptr)); + fused_schemas_containers_.push_back(function_utils::CreateSchema(*this, sub_graph)); fused_node.op_ = fused_schemas_containers_.back().get(); } #endif @@ -3831,31 +3862,6 @@ Node& Graph::BeginFuseSubGraph(const IndexedSubGraph& sub_graph, const std::stri return node; } -void Graph::CancelFuseSubGraph(const Node& fused_node) { - auto node_idx = fused_node.Index(); - if (!GetNode(node_idx)) - return; - - if (fused_node.NodeType() != Node::Type::Fused) - return; - -#if !defined(ORT_MINIMAL_BUILD) - // Remove the tempoary schema from schema container container - auto* temp_schema_ptr = fused_node.Op(); - auto it = std::find_if( - fused_schemas_containers_.begin(), fused_schemas_containers_.end(), - [temp_schema_ptr](const std::unique_ptr& schema) { - return schema.get() == temp_schema_ptr; - }); - if (it != fused_schemas_containers_.end()) { - fused_schemas_containers_.erase(it); - } -#endif - - // Remove the fused_node - RemoveNode(node_idx); -} - void Graph::FinalizeFuseSubGraph(const IndexedSubGraph& sub_graph, Node& fused_node) { const auto* func_meta_def = sub_graph.GetMetaDef(); ORT_ENFORCE(nullptr != func_meta_def); diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 88158ca6d4..20c0275bff 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -64,6 +64,7 @@ #include "core/optimizer/unsqueeze_elimination.h" #ifdef ENABLE_TRAINING #include "orttraining/core/optimizer/bitmask_dropout_replacement.h" +#include "orttraining/core/optimizer/bias_softmax_dropout_fusion.h" #endif #endif // !defined(ORT_MINIMAL_BUILD) @@ -248,13 +249,15 @@ InlinedVector> GenerateTransformers( transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cuda_rocm_eps)); -#ifdef ENABLE_TRAINING - transformers.emplace_back(std::make_unique(cuda_rocm_eps)); -#endif transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(std::make_unique(cuda_rocm_eps)); +#ifdef ENABLE_TRAINING + transformers.emplace_back(std::make_unique(cuda_rocm_eps)); + transformers.emplace_back(std::make_unique(cuda_rocm_eps)); +#endif + transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); diff --git a/onnxruntime/core/optimizer/initializer.cc b/onnxruntime/core/optimizer/initializer.cc index 1128fadcd0..50092d2fb8 100644 --- a/onnxruntime/core/optimizer/initializer.cc +++ b/onnxruntime/core/optimizer/initializer.cc @@ -36,8 +36,7 @@ Initializer::Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, const name_ = tensor_proto.name(); } - auto proto_dims = utils::GetTensorShapeFromTensorProto(tensor_proto); - TensorShape proto_shape(proto_dims); + auto proto_shape = utils::GetTensorShapeFromTensorProto(tensor_proto); // This must be pre-allocated Tensor w(DataTypeImpl::TensorTypeFromONNXEnum(proto_data_type)->GetElementType(), proto_shape, std::make_shared()); @@ -45,6 +44,7 @@ Initializer::Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, const data_ = std::move(w); } +#if !defined(ORT_EXTENDED_MINIMAL_BUILD) namespace { template struct ToFp16; @@ -319,5 +319,5 @@ void Initializer::scale_by_axis(const Initializer& scalers, int axis) { utils::MLTypeCallDispatcher t_disp(data_.GetElementType()); t_disp.Invoke(data_, scalers.data_, block_size, num_blocks); } - +#endif // ORT_EXTENDED_MINIMAL_BUILD } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/initializer.h b/onnxruntime/core/optimizer/initializer.h index f5ee267370..47ade74d2e 100644 --- a/onnxruntime/core/optimizer/initializer.h +++ b/onnxruntime/core/optimizer/initializer.h @@ -27,18 +27,18 @@ class Initializer final { gsl::span dims); Initializer(const ONNX_NAMESPACE::TensorProto& tensor_proto, - const Path& model_path); + const Path& model_path = {}); ~Initializer() = default; void ToProto(ONNX_NAMESPACE::TensorProto& tensor_proto) const { tensor_proto = utils::TensorToTensorProto(data_, name_); } - +#if !defined(ORT_EXTENDED_MINIMAL_BUILD) ONNX_NAMESPACE::TensorProto ToFP16(const std::string& name) const; ONNX_NAMESPACE::TensorProto ToBFloat16(const std::string& name) const; - +#endif // ORT_EXTENDED_MINIMAL_BUILD int data_type() const { return data_.GetElementType(); } @@ -57,8 +57,13 @@ class Initializer final { return data_.Data(); } - const int8_t* raw_data() const { - return reinterpret_cast(data_.DataRaw()); + template + auto DataAsSpan() const { + return data_.DataAsSpan(); + } + + gsl::span DataAsByteSpan() const { + return gsl::make_span(reinterpret_cast(data_.DataRaw()), data_.SizeInBytes()); } gsl::span dims() const { @@ -67,12 +72,13 @@ class Initializer final { int64_t size() const { return data_.Shape().Size(); } +#if !defined(ORT_EXTENDED_MINIMAL_BUILD) Initializer& add(float value); Initializer& add(const Initializer& other); Initializer& sub(const Initializer& other); - + Initializer& mul(const Initializer& other); Initializer& div(const Initializer& other); @@ -80,7 +86,7 @@ class Initializer final { Initializer& sqrt(); void scale_by_axis(const Initializer& other, int axis); - +#endif // ORT_EXTENDED_MINIMAL_BUILD private: std::string name_; diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc index d595d9e35e..1898b6b290 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc @@ -54,7 +54,7 @@ bool IsQDQPairSupported( Initializer dq_scale(*dq_scale_tensor_proto, model_path); return q_zp.data_type() == dq_zp.data_type() && - *q_zp.raw_data() == *dq_zp.raw_data() && + q_zp.DataAsByteSpan() == dq_zp.DataAsByteSpan() && *q_scale.data() == *dq_scale.data(); } diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.cc index 379f90a4f0..849ad87521 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.cc @@ -73,6 +73,15 @@ std::vector ConvMoves() { return moves; } +QDQReplaceWithNew SplitReplacer() { + NTO::NodeLocation dq{NTO::NodeType::kInput, 0}; + NTO::NodeLocation q{NTO::NodeType::kOutput, 0}; + std::vector moves{ + MoveAndAppend(dq, ArgType::kInput, 0, ArgType::kInput), + MoveAll(q, ArgType::kOutput)}; + return QDQReplaceWithNew(kOnnxDomain, "Split", std::move(moves)); +} + QDQReplaceWithNew MatMulIntToFloatReplacer() { NTO::NodeLocation dq1{NTO::NodeType::kInput, 0}; NTO::NodeLocation dq2{NTO::NodeType::kInput, 1}; @@ -221,6 +230,10 @@ MatMulReplaceWithQLinear::MatMulReplaceWithQLinear() qlinear_matmul_replacer_{kOnnxDomain} { } +Status SplitReplaceWithQuant::Run(Graph& graph, const NodesToOptimize& selected_nodes) const { + return SplitReplacer().Run(graph, selected_nodes); +} + Status MatMulReplaceWithQLinear::Run(Graph& graph, const NodesToOptimize& selected_nodes) const { // if the output is empty there were no Q nodes selected, so replace with MatMulIntegerToFloat // otherwise replace with QLinearMatMul diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.h b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.h index b8e371ecc8..0337f11306 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.h +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_actions.h @@ -60,6 +60,10 @@ struct ConvReplaceWithQLinear : ReplaceWithQLinear { ConvReplaceWithQLinear(); }; +struct SplitReplaceWithQuant : public Action { + Status Run(Graph&, const NodesToOptimize& selected_nodes) const override; +}; + struct MatMulReplaceWithQLinear : public Action { MatMulReplaceWithQLinear(); diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc index a2ea67813f..54ba592144 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc @@ -15,6 +15,20 @@ namespace { using NTO = onnxruntime::NodesToOptimize; +void SplitQDQRules(SelectorActionRegistry& qdq_selector_action_registry) { + const std::string action_name{"dropSplitQDQ"}; + std::unique_ptr action = std::make_unique(); +#if !defined(ORT_MINIMAL_BUILD) + std::unique_ptr selector = std::make_unique(); + qdq_selector_action_registry.RegisterSelectorAndAction(action_name, + {{"Split", {}}}, + std::move(selector), + std::move(action)); +#else + qdq_selector_action_registry.RegisterAction(action_name, std::move(action)); +#endif +} + // create rules for ops that don't change the data void DropQDQNodesRules(SelectorActionRegistry& qdq_selector_action_registry) { // 3 nodes. DQ, target, Q. Merge into target and remove DQ and Q. @@ -117,7 +131,7 @@ void VariadicOpQDQRules(SelectorActionRegistry& qdq_selector_action_registry) { std::unique_ptr action = std::make_unique(kMSDomain); #if !defined(ORT_MINIMAL_BUILD) - std::unique_ptr selector = std::make_unique(); + std::unique_ptr selector = std::make_unique(); qdq_selector_action_registry.RegisterSelectorAndAction(action_name, {{"Concat", {}}}, @@ -194,7 +208,7 @@ void GemmQDQRules(SelectorActionRegistry& qdq_selector_action_registry) { SelectorActionRegistry CreateSelectorActionRegistry(bool is_int8_allowed) { SelectorActionRegistry qdq_selector_action_registry; - + SplitQDQRules(qdq_selector_action_registry); DropQDQNodesRules(qdq_selector_action_registry); DropDQNodesRules(qdq_selector_action_registry); UnaryOpQDQRules(qdq_selector_action_registry); diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc index 3722504695..4972d749bf 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc @@ -180,13 +180,22 @@ bool VariadicNodeGroupSelector::Check(const GraphViewer& graph_viewer, } int32_t dt_output = q_nodes[0]->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + for (size_t q_idx = 1; q_idx < q_nodes.size(); q_idx++) { + if (dt_output != q_nodes[q_idx]->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type()) { + return false; + } + } return dt_input == dt_output; } -void VariadicSelector::UpdateBuilder(NodesToOptimizeIndicesBuilder& builder) const { +void InputVariadicSelector::UpdateBuilder(NodesToOptimizeIndicesBuilder& builder) const { builder.num_input_defs = 1; // set to 1 as the first input is variadic } +void OutputVariadicSelector::UpdateBuilder(NodesToOptimizeIndicesBuilder& builder) const { + builder.num_output_defs = 1; // set to 1 as the first output is variadic +} + bool ConvNodeGroupSelector::Check(const GraphViewer& graph_viewer, const Node& node, const std::vector& dq_nodes, diff --git a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h index ddafc20d0b..4f52f8493b 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h +++ b/onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h @@ -178,9 +178,17 @@ class BinarySelector : public BaseSelector { }; // Variadic DQ nodes -> node -> Q -class VariadicSelector : public BaseSelector { +class InputVariadicSelector : public BaseSelector { public: - VariadicSelector() : BaseSelector(std::make_unique()) {} + InputVariadicSelector() : BaseSelector(std::make_unique()) {} + + void UpdateBuilder(NodesToOptimizeIndicesBuilder&) const override; +}; + +// DQ -> node -> Variadic Q nodes +class OutputVariadicSelector : public BaseSelector { + public: + OutputVariadicSelector() : BaseSelector(std::make_unique()) {} void UpdateBuilder(NodesToOptimizeIndicesBuilder&) const override; }; diff --git a/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc b/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc index 4c882ff1c3..a002d108f8 100644 --- a/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc +++ b/onnxruntime/core/optimizer/transpose_optimizer/optimizer_api_impl.cc @@ -244,11 +244,13 @@ void ApiValueInfo::UnsqueezeDims(const std::vector& axes) { // std::vector ApiTensor::Shape() const { - return utils::GetTensorShapeFromTensorProto(tensor_proto_); + TensorShape shape = utils::GetTensorShapeFromTensorProto(tensor_proto_); + const auto dims = shape.GetDims(); + return std::vector{dims.cbegin(), dims.cend()}; } size_t ApiTensor::NumElements() const { - int64_t size = TensorShape(utils::GetTensorShapeFromTensorProto(tensor_proto_)).Size(); + int64_t size = utils::GetTensorShapeFromTensorProto(tensor_proto_).Size(); ORT_ENFORCE(size >= 0, "Failed to get size of TensorProto"); return gsl::narrow_cast(size); } diff --git a/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc b/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc index 81adcee7cb..2cac02033e 100644 --- a/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc +++ b/onnxruntime/core/optimizer/transpose_optimizer/transpose_optimizer.cc @@ -1691,7 +1691,19 @@ static const std::unordered_map handler_ma {"Split", split_handler}, {"Shape", shape_handler}, {"Pad", pad_handler}, +// The CUDA Resize kernel assumes that the input is NCHW and +// Resize can't be supported in ORT builds with CUDA enabled. +// TODO: Enable this once the CUDA Resize kernel is implemented +// "generically" (i.e.) aligning with the generic nature of the +// ONNX spec. +// See https://github.com/microsoft/onnxruntime/pull/10824 for +// a similar fix applied to the CPU Resize kernel. +// Per tests included in #10824, the ROCM EP also generates +// incorrect results when this handler is used, so the Resize +// handler is not enabled even for those builds. +#if !defined(USE_CUDA) && !defined(USE_ROCM) {"Resize", resize_handler}, +#endif {"ReduceSum", reduce_sum_handler}, {"ReduceLogSum", reduce_op_handler}, diff --git a/onnxruntime/core/platform/windows/stacktrace.cc b/onnxruntime/core/platform/windows/stacktrace.cc index ca3aa72b85..4a802f0cc9 100644 --- a/onnxruntime/core/platform/windows/stacktrace.cc +++ b/onnxruntime/core/platform/windows/stacktrace.cc @@ -33,7 +33,7 @@ class CaptureStackTrace { std::vector GetStackTrace() { #ifndef NDEBUG // TVM need to run with shared CRT, so won't work with debug helper now -#if !(defined USE_NUPHAR_TVM) && !(defined _OPSCHEMA_LIB_) && !(defined _GAMING_XBOX) +#if !(defined _OPSCHEMA_LIB_) && !(defined _GAMING_XBOX) return detail::CaptureStackTrace().Trace(); #else return {}; @@ -45,7 +45,7 @@ std::vector GetStackTrace() { namespace detail { #ifndef NDEBUG -#if !(defined USE_NUPHAR_TVM) && !(defined _OPSCHEMA_LIB_) && !(defined _GAMING_XBOX) +#if !(defined _OPSCHEMA_LIB_) && !(defined _GAMING_XBOX) class SymbolHelper { public: SymbolHelper() noexcept { diff --git a/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc index 37993f3e8c..f15618f821 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/activation_op_builder.cc @@ -10,6 +10,7 @@ #include "core/providers/coreml/builders/helper.h" #include "core/providers/coreml/builders/impl/base_op_builder.h" #include "core/providers/coreml/builders/op_builder_factory.h" +#include "core/optimizer/initializer.h" namespace onnxruntime { namespace coreml { @@ -67,10 +68,8 @@ Status AddPReluWeight(ModelBuilder& model_builder, const Node& node, // assume X has 3 or 4 dimensions, that was checked in IsPReluOpSupported() const auto num_channels = x_shape[x_shape.size() - 3]; - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(slope_tensor, unpacked_tensor)); - float value; - std::memcpy(&value, unpacked_tensor.data(), sizeof(value)); + Initializer unpacked_tensor(slope_tensor); + float value = unpacked_tensor.DataAsSpan()[0]; auto& weight_values = *prelu.mutable_alpha()->mutable_floatvalue(); weight_values.Clear(); diff --git a/onnxruntime/core/providers/coreml/builders/impl/base_op_builder.h b/onnxruntime/core/providers/coreml/builders/impl/base_op_builder.h index e8ec01f34c..059e767f4a 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/base_op_builder.h +++ b/onnxruntime/core/providers/coreml/builders/impl/base_op_builder.h @@ -46,7 +46,7 @@ class BaseOpBuilder : public IOpBuilder { virtual bool HasSupportedInputsImpl(const Node& node, const logging::Logger& logger) const; virtual int GetMinSupportedOpSet(const Node& /* node */) const { return 1; } - virtual int GetMaxSupportedOpSet(const Node& /* node */) const { return 15; } + virtual int GetMaxSupportedOpSet(const Node& /* node */) const { return 17; } private: bool HasSupportedOpSet(const Node& node, const logging::Logger& logger) const; @@ -54,4 +54,4 @@ class BaseOpBuilder : public IOpBuilder { }; } // namespace coreml -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc index 9374082390..d14c6ff041 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc @@ -9,6 +9,7 @@ #include "core/framework/tensorprotoutils.h" #include "core/providers/coreml/builders/helper.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" #include "coreml/NeuralNetwork.pb.h" @@ -94,10 +95,9 @@ common::Status CreateCoreMLWeight(CoreML::Specification::WeightParams& weight, const ONNX_NAMESPACE::TensorProto& tensor) { auto data_type = tensor.data_type(); if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor)); + Initializer unpacked_tensor(tensor); auto num_elements = SafeInt(Product(tensor.dims())); - CreateCoreMLWeight(weight, reinterpret_cast(unpacked_tensor.data()), num_elements); + CreateCoreMLWeight(weight, unpacked_tensor.data(), num_elements); } else { // TODO: support other type return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, diff --git a/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc index a2f5d47265..8d675050fa 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/gemm_op_builder.cc @@ -7,6 +7,8 @@ #include "core/providers/shared/utils/utils.h" #include "core/providers/coreml/builders/helper.h" #include "core/providers/coreml/builders/op_builder_factory.h" +#include "core/optimizer/initializer.h" + #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" #include "builder_utils.h" @@ -52,9 +54,8 @@ void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Nod // TODO, add support of other data types static Status GetTensorFloatDataTransposed(const ONNX_NAMESPACE::TensorProto& tensor, std::vector& transposed_data) { - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(tensor, unpacked_tensor)); - const float* src_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(tensor); + auto src_data = unpacked_tensor.DataAsSpan(); const auto& tensor_shape = tensor.dims(); auto x_t = SafeInt(tensor_shape[0]); auto y_t = SafeInt(tensor_shape[1]); diff --git a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc index 19bff2e97d..86318e8b2b 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/reshape_op_builder.cc @@ -4,6 +4,7 @@ #include "core/providers/common.h" #include "core/framework/tensorprotoutils.h" #include "core/providers/cpu/tensor/reshape_helper.h" +#include "core/optimizer/initializer.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/coreml/builders/helper.h" @@ -83,14 +84,8 @@ bool ReshapeOpBuilder::IsOpSupportedImpl(const Node& node, const OpBuilderInputP } const auto& perm_tensor = *initializers.at(perm_name); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage(); - return false; - } - - const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(perm_tensor); + auto raw_perm = unpacked_tensor.DataAsSpan(); const auto& perm_dims = perm_tensor.dims(); if (perm_dims.empty() || perm_dims[0] == 0) { LOGS(logger, VERBOSE) << "New shape of reshape cannot be empty"; diff --git a/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc index eb52bbf7b2..d35ef21e27 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/resize_op_builder.cc @@ -8,6 +8,7 @@ #include "core/providers/coreml/builders/helper.h" #include "core/providers/cpu/tensor/reshape_helper.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" @@ -43,7 +44,7 @@ class ResizeOpBuilder : public BaseOpBuilder { // Helper functions bool GetResizeScales(const InitializedTensorSet& initializers, const Node& node, std::vector& scales, - const logging::Logger& logger) { + const logging::Logger&) { const auto& input_defs = node.InputDefs(); if (input_defs.size() < 3) return false; @@ -51,21 +52,15 @@ bool GetResizeScales(const InitializedTensorSet& initializers, const auto& scales_tensor = *initializers.at(input_defs[2]->Name()); if (scales_tensor.dims_size() != 1 || scales_tensor.dims()[0] != 4) return false; - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage(); - return false; - } - const float* scales_data = reinterpret_cast(unpacked_tensor.data()); - scales = std::vector{scales_data, scales_data + 4}; + Initializer unpacked_tensor(scales_tensor); + auto scales_data = unpacked_tensor.DataAsSpan(); + scales = std::vector{scales_data.begin(), scales_data.end()}; return true; } bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const Node& node, std::vector& sizes, - const logging::Logger& logger) { + const logging::Logger&) { const auto& input_defs = node.InputDefs(); if (input_defs.size() < 4) return false; @@ -73,15 +68,9 @@ bool GetResizeOutputSizes(const InitializedTensorSet& initializers, const auto& sizes_tensor = *initializers.at(input_defs[3]->Name()); if (sizes_tensor.dims_size() != 1 || sizes_tensor.dims()[0] != 4) return false; - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS(logger, ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage(); - return false; - } - const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); - sizes = std::vector{sizes_data, sizes_data + 4}; + Initializer unpacked_tensor(sizes_tensor); + auto sizes_data = unpacked_tensor.DataAsSpan(); + sizes = std::vector{sizes_data.begin(), sizes_data.end()}; return true; } diff --git a/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc b/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc index 2c5325daf9..0665d8e334 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/squeeze_op_builder.cc @@ -4,6 +4,8 @@ #include "core/framework/tensorprotoutils.h" #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" + #ifdef __APPLE__ #include "core/providers/coreml/builders/model_builder.h" #endif @@ -47,14 +49,11 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const if (node.InputDefs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& axes_tensor = *initializers.at(node.InputDefs()[1]->Name()); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); - const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(axes_tensor); + auto raw_axes = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(axes_tensor.dims()[0]); - axes.resize(size); - for (size_t i = 0; i < size; i++) { - axes[i] = raw_axes[i]; - } + axes.reserve(size); + axes.insert(axes.end(), raw_axes.begin(), raw_axes.end()); } } else { NodeAttrHelper helper(node); diff --git a/onnxruntime/core/providers/coreml/model/model.mm b/onnxruntime/core/providers/coreml/model/model.mm index 06a2a8e935..37c4a2d227 100644 --- a/onnxruntime/core/providers/coreml/model/model.mm +++ b/onnxruntime/core/providers/coreml/model/model.mm @@ -17,6 +17,10 @@ #import #import +// force the linker to create a dependency on the CoreML framework so that in MAUI usage we don't need +// to manually do this +asm(".linker_option \"-framework\", \"CoreML\""); + // Model input for a CoreML model // All the input onnx tensors values will be converted to MLMultiArray(s) @interface OnnxTensorFeatureProvider : NSObject { diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 191f34439c..c627c7bf3c 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -791,7 +791,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 17, ST opset. To double-check what versions an operator should have registrations for see - https://github.com/onnx/onnx/blob/master/docs/Operators.md + https://github.com/onnx/onnx/blob/main/docs/Operators.md *****/ template <> diff --git a/onnxruntime/core/providers/cpu/generator/random.cc b/onnxruntime/core/providers/cpu/generator/random.cc index 0b36adbaf6..6492b2ca63 100644 --- a/onnxruntime/core/providers/cpu/generator/random.cc +++ b/onnxruntime/core/providers/cpu/generator/random.cc @@ -132,7 +132,7 @@ ONNX_CPU_OPERATOR_KERNEL( BuildKernelDefConstraintsFromTypeList()), RandomUniformLike); -// https://github.com/onnx/onnx/blob/master/docs/Operators.md#multinomial +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#multinomial ONNX_CPU_OPERATOR_KERNEL( Multinomial, 7, diff --git a/onnxruntime/core/providers/cpu/math/hardmax.cc b/onnxruntime/core/providers/cpu/math/hardmax.cc index a28ad99682..6630626488 100644 --- a/onnxruntime/core/providers/cpu/math/hardmax.cc +++ b/onnxruntime/core/providers/cpu/math/hardmax.cc @@ -30,8 +30,8 @@ Status Hardmax::Compute(OpKernelContext* ctx) const { std::vector permutation(rank); // The "semantic" meaning of axis has changed in opset-13. - // Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Hardmax - // with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Hardmax-11 for detailed explanations + // Please compare: https://github.com/onnx/onnx/blob/main/docs/Operators.md#Hardmax + // with https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Hardmax-11 for detailed explanations // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already // the innermost dim diff --git a/onnxruntime/core/providers/cpu/math/softmax.cc b/onnxruntime/core/providers/cpu/math/softmax.cc index 6526b5f760..a249c93944 100644 --- a/onnxruntime/core/providers/cpu/math/softmax.cc +++ b/onnxruntime/core/providers/cpu/math/softmax.cc @@ -133,8 +133,8 @@ Status Softmax::ComputeImplOpset13(const Tensor& input, Tensor& output, size_ std::vector permutation(rank); // The "semantic" meaning of axis has changed in opset-13. - // Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax - // with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations + // Please compare: https://github.com/onnx/onnx/blob/main/docs/Operators.md#Softmax + // with https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Softmax-11 for detailed explanations // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already // the innermost dim diff --git a/onnxruntime/core/providers/cpu/math/top_k.cc b/onnxruntime/core/providers/cpu/math/top_k.cc index 641f64e853..b666621d31 100644 --- a/onnxruntime/core/providers/cpu/math/top_k.cc +++ b/onnxruntime/core/providers/cpu/math/top_k.cc @@ -572,7 +572,7 @@ Status TopK<11, int64_t>::Compute(OpKernelContext* p_op_kernel_context) const { } // Register necessary kernels -// spec https://github.com/onnx/onnx/blob/master/docs/Operators.md#TopK +// spec https://github.com/onnx/onnx/blob/main/docs/Operators.md#TopK #define REGISTER_TOPK_VERSIONED_TYPED_KERNEL(OPSET1, OPSET2, TYPE) \ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(TopK, OPSET1, OPSET2, TYPE, \ diff --git a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc index 5743a9e03a..af67419f4f 100644 --- a/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc +++ b/onnxruntime/core/providers/cpu/ml/array_feature_extractor.cc @@ -4,7 +4,7 @@ #include "core/providers/cpu/ml/array_feature_extractor.h" /** -https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +https://github.com/onnx/onnx/blob/main/onnx/defs/traditionalml/defs.cc ONNX_OPERATOR_SCHEMA(ArrayFeatureExtractor) .SetDomain("ai.onnx.ml") .SetDoc(R"DOC( diff --git a/onnxruntime/core/providers/cpu/ml/binarizer.cc b/onnxruntime/core/providers/cpu/ml/binarizer.cc index f577f48105..0c7af89b5f 100644 --- a/onnxruntime/core/providers/cpu/ml/binarizer.cc +++ b/onnxruntime/core/providers/cpu/ml/binarizer.cc @@ -4,7 +4,7 @@ #include "core/providers/cpu/ml/binarizer.h" #include /** -https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +https://github.com/onnx/onnx/blob/main/onnx/defs/traditionalml/defs.cc ONNX_OPERATOR_SCHEMA(Binarizer) .SetDomain("ai.onnx.ml") diff --git a/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h b/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h index b1b4b9a0f1..ea92e062cd 100644 --- a/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h +++ b/onnxruntime/core/providers/cpu/ml/feature_vectorizer.h @@ -10,7 +10,7 @@ namespace onnxruntime { namespace ml { -// https://github.com/onnx/onnx/blob/master/docs/Operators-ml.md#ai.onnx.ml.FeatureVectorizer +// https://github.com/onnx/onnx/blob/main/docs/Operators-ml.md#ai.onnx.ml.FeatureVectorizer class FeatureVectorizer final : public OpKernel { public: FeatureVectorizer(const OpKernelInfo& info) : OpKernel(info) { diff --git a/onnxruntime/core/providers/cpu/ml/imputer.cc b/onnxruntime/core/providers/cpu/ml/imputer.cc index 5f310ea11b..5477f14d3f 100644 --- a/onnxruntime/core/providers/cpu/ml/imputer.cc +++ b/onnxruntime/core/providers/cpu/ml/imputer.cc @@ -4,7 +4,7 @@ #include "core/providers/cpu/ml/imputer.h" #include /** -https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +https://github.com/onnx/onnx/blob/main/onnx/defs/traditionalml/defs.cc ONNX_OPERATOR_SCHEMA(Imputer) .SetDomain("ai.onnx.ml") .SetDoc(R"DOC( diff --git a/onnxruntime/core/providers/cpu/ml/linearclassifier.cc b/onnxruntime/core/providers/cpu/ml/linearclassifier.cc index 83b6076ebd..e55ebc36b5 100644 --- a/onnxruntime/core/providers/cpu/ml/linearclassifier.cc +++ b/onnxruntime/core/providers/cpu/ml/linearclassifier.cc @@ -38,7 +38,7 @@ LinearClassifier::LinearClassifier(const OpKernelInfo& info) } // Use GEMM for the calculations, with broadcasting of intercepts -// https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gemm +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#Gemm // // X: [num_batches, num_features] // coefficients_: [num_targets, num_features] diff --git a/onnxruntime/core/providers/cpu/ml/linearregressor.cc b/onnxruntime/core/providers/cpu/ml/linearregressor.cc index 0f109848c4..aedee95c7e 100644 --- a/onnxruntime/core/providers/cpu/ml/linearregressor.cc +++ b/onnxruntime/core/providers/cpu/ml/linearregressor.cc @@ -28,7 +28,7 @@ LinearRegressor::LinearRegressor(const OpKernelInfo& info) } // Use GEMM for the calculations, with broadcasting of intercepts -// https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gemm +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#Gemm // // X: [num_batches, num_features] // coefficients_: [num_targets, num_features] diff --git a/onnxruntime/core/providers/cpu/ml/onehotencoder.cc b/onnxruntime/core/providers/cpu/ml/onehotencoder.cc index 9b3b6be369..37db277f9e 100644 --- a/onnxruntime/core/providers/cpu/ml/onehotencoder.cc +++ b/onnxruntime/core/providers/cpu/ml/onehotencoder.cc @@ -3,7 +3,7 @@ #include "core/providers/cpu/ml/onehotencoder.h" /** -https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +https://github.com/onnx/onnx/blob/main/onnx/defs/traditionalml/defs.cc ONNX_OPERATOR_SCHEMA(OneHotEncoder) .SetDomain("ai.onnx.ml") .SetDoc(R"DOC( diff --git a/onnxruntime/core/providers/cpu/ml/scaler.cc b/onnxruntime/core/providers/cpu/ml/scaler.cc index 1361fcb153..84317f823f 100644 --- a/onnxruntime/core/providers/cpu/ml/scaler.cc +++ b/onnxruntime/core/providers/cpu/ml/scaler.cc @@ -4,7 +4,7 @@ #include "core/providers/cpu/ml/scaler.h" /** -https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +https://github.com/onnx/onnx/blob/main/onnx/defs/traditionalml/defs.cc ONNX_OPERATOR_SCHEMA(Scaler) .SetDomain("ai.onnx.ml") .SetDoc(R"DOC( diff --git a/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h b/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h index 7db1c5c0c7..902eb1df6b 100644 --- a/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h +++ b/onnxruntime/core/providers/cpu/ml/tree_ensemble_common.h @@ -25,6 +25,7 @@ class TreeEnsembleCommonAttributes { AGGREGATE_FUNCTION aggregate_function_; int64_t n_nodes_; int64_t max_tree_depth_; + int64_t max_feature_id_; int64_t n_trees_; bool same_mode_; bool has_missing_tracks_; @@ -196,12 +197,16 @@ Status TreeEnsembleCommon::Init(int parall nodes_.resize(n_nodes_); roots_.clear(); std::unordered_map*, TreeNodeElementId::hash_fn> idi; + max_feature_id_ = 0; for (i = 0, limit = nodes_treeids.size(); i < limit; ++i) { TreeNodeElement& node = nodes_[i]; node.id.tree_id = static_cast(nodes_treeids[i]); node.id.node_id = static_cast(nodes_nodeids[i]); node.feature_id = static_cast(nodes_featureids[i]); + if (node.feature_id > max_feature_id_) { + max_feature_id_ = node.feature_id; + } if (nodes_values_as_tensor.empty()) { node.value = static_cast(nodes_values[i]); } else { @@ -344,8 +349,15 @@ template void TreeEnsembleCommon::ComputeAgg(concurrency::ThreadPool* ttp, const Tensor* X, Tensor* Z, Tensor* label, const AGG& agg) const { + if (X->Shape().NumDimensions() > 2) { + ORT_THROW("TreeEnsemble only works on 1D, 2D tensors."); + } int64_t stride = X->Shape().NumDimensions() == 1 ? X->Shape()[0] : X->Shape()[1]; int64_t N = X->Shape().NumDimensions() == 1 ? 1 : X->Shape()[0]; + int64_t C = X->Shape().NumDimensions() == 2 ? X->Shape()[1] : 1; + if (max_feature_id_ >= C) { + ORT_THROW("One path in the graph requests feature ", max_feature_id_, " but input tensor has ", C, " features."); + } OutputType* z_data = Z->MutableData(); const InputType* x_data = X->Data(); diff --git a/onnxruntime/core/providers/cpu/ml/zipmap.cc b/onnxruntime/core/providers/cpu/ml/zipmap.cc index 0b6f82b9f1..76993e5921 100644 --- a/onnxruntime/core/providers/cpu/ml/zipmap.cc +++ b/onnxruntime/core/providers/cpu/ml/zipmap.cc @@ -4,7 +4,7 @@ #include "core/providers/cpu/ml/zipmap.h" #include "core/util/math_cpuonly.h" /** -https://github.com/onnx/onnx/blob/master/onnx/defs/traditionalml/defs.cc +https://github.com/onnx/onnx/blob/main/onnx/defs/traditionalml/defs.cc ONNX_OPERATOR_SCHEMA(ZipMap) .SetDomain("ai.onnx.ml") .SetDoc(R"DOC( diff --git a/onnxruntime/core/providers/cpu/nn/batch_norm.cc b/onnxruntime/core/providers/cpu/nn/batch_norm.cc index ebd1af6c0e..46e812efb4 100644 --- a/onnxruntime/core/providers/cpu/nn/batch_norm.cc +++ b/onnxruntime/core/providers/cpu/nn/batch_norm.cc @@ -19,7 +19,7 @@ #include "core/providers/cpu/nn/batch_norm_helper.h" namespace onnxruntime { -// spec: https://github.com/onnx/onnx/blob/master/docs/Operators.md#BatchNormalization +// spec: https://github.com/onnx/onnx/blob/main/docs/Operators.md#BatchNormalization ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(BatchNormalization, 7, 8, float, KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose.h b/onnxruntime/core/providers/cpu/nn/conv_transpose.h index 53025e3b13..49c174969b 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_transpose.h +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose.h @@ -25,15 +25,7 @@ namespace onnxruntime { template class ConvTranspose : public OpKernel { public: - ConvTranspose(const OpKernelInfo& info) : OpKernel(info), conv_transpose_attrs_(info) { - if (conv_transpose_attrs_.auto_pad == AutoPadType::SAME_UPPER || - conv_transpose_attrs_.auto_pad == AutoPadType::SAME_LOWER) { - // TODO(jcwchen): #9740 ORT 1.13 will correct the logic by switching them to meet ONNX spec - LOGS_DEFAULT(WARNING) << "The existing bug in the padding distribution for auto_pad type" - << " SAME_UPPER/SAME_LOWER will be fixed in next ORT 1.13 release and hence the" - << " results of ConvTranspose operator using the above auto_pad type(s) will be different."; - } - } + ConvTranspose(const OpKernelInfo& info) : OpKernel(info), conv_transpose_attrs_(info) {} Status PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, /*out*/ bool& is_packed, diff --git a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h index ee74877da8..7d0bc28708 100644 --- a/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h @@ -176,7 +176,7 @@ struct ConvTransposeAttributes : public ConvAttributes { void DistributePadding(AutoPadType pad_type, const int64_t& total_pad, int64_t& pad_head, int64_t& pad_tail) const { - if (pad_type == AutoPadType::SAME_UPPER) { // pad more on head when total_pad is odd. + if (pad_type == AutoPadType::SAME_LOWER) { // pad more on head when total_pad is odd. pad_head = total_pad - total_pad / 2; pad_tail = total_pad / 2; } else { diff --git a/onnxruntime/core/providers/cpu/nn/pool_attributes.h b/onnxruntime/core/providers/cpu/nn/pool_attributes.h index 392936f0b8..500750da1b 100644 --- a/onnxruntime/core/providers/cpu/nn/pool_attributes.h +++ b/onnxruntime/core/providers/cpu/nn/pool_attributes.h @@ -23,7 +23,6 @@ struct PoolAttributes { // Shared providers don't know about OpNodeProtoHelper PoolAttributes(const OpKernelInfo& info, #else - // Providers like Nuphar don't know about OpKernelInfo PoolAttributes(const OpNodeProtoHelper& info, #endif const std::string& op_name, int start_version) diff --git a/onnxruntime/core/providers/cpu/tensor/gather.cc b/onnxruntime/core/providers/cpu/tensor/gather.cc index 9d81059e3b..fd499e3653 100644 --- a/onnxruntime/core/providers/cpu/tensor/gather.cc +++ b/onnxruntime/core/providers/cpu/tensor/gather.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//https://github.com/onnx/onnx/blob/master/docs/Operators.md#Gather +//https://github.com/onnx/onnx/blob/main/docs/Operators.md#Gather #include "core/providers/cpu/tensor/gather.h" #include "core/common/common.h" #include "core/framework/op_kernel_type_control_utils.h" diff --git a/onnxruntime/core/providers/cpu/tensor/isinf.cc b/onnxruntime/core/providers/cpu/tensor/isinf.cc index 88144256c9..104da86df7 100644 --- a/onnxruntime/core/providers/cpu/tensor/isinf.cc +++ b/onnxruntime/core/providers/cpu/tensor/isinf.cc @@ -11,7 +11,7 @@ #include "core/providers/op_kernel_type_control.h" namespace onnxruntime { -// https://github.com/onnx/onnx/blob/master/docs/Operators.md#IsInf +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#IsInf namespace op_kernel_type_control { ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES_ALL_OPSETS( diff --git a/onnxruntime/core/providers/cpu/tensor/isnan.cc b/onnxruntime/core/providers/cpu/tensor/isnan.cc index 047855304b..a22d619709 100644 --- a/onnxruntime/core/providers/cpu/tensor/isnan.cc +++ b/onnxruntime/core/providers/cpu/tensor/isnan.cc @@ -9,7 +9,7 @@ #include "Eigen/src/Core/arch/Default/Half.h" namespace onnxruntime { -// https://github.com/onnx/onnx/blob/master/docs/Operators.md#IsNaN +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#IsNaN #define ADD_TYPED_ISNAN_OP_9(data_type) \ ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL( \ IsNaN, \ diff --git a/onnxruntime/core/providers/cpu/tensor/onehot.cc b/onnxruntime/core/providers/cpu/tensor/onehot.cc index 49dc8b627a..48de8629f6 100644 --- a/onnxruntime/core/providers/cpu/tensor/onehot.cc +++ b/onnxruntime/core/providers/cpu/tensor/onehot.cc @@ -26,7 +26,7 @@ using namespace ::onnxruntime::common; using namespace std; namespace onnxruntime { -// spec: https://github.com/onnx/onnx/blob/master/docs/Operators.md#OneHot +// spec: https://github.com/onnx/onnx/blob/main/docs/Operators.md#OneHot // T1: indices, T2: depth, T3: values #define REG_TYPED_ONE_HOT_OP_V9_10(types_str, in_type, out_type, depth_type) \ diff --git a/onnxruntime/core/providers/cpu/tensor/scatter.cc b/onnxruntime/core/providers/cpu/tensor/scatter.cc index 4c0957bbe9..ae0fe2d5ba 100644 --- a/onnxruntime/core/providers/cpu/tensor/scatter.cc +++ b/onnxruntime/core/providers/cpu/tensor/scatter.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -//https://github.com/onnx/onnx/blob/master/docs/Operators.md#Scatter +//https://github.com/onnx/onnx/blob/main/docs/Operators.md#Scatter #include #include "gsl/gsl" diff --git a/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh b/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh index 1f76a6c096..2949d1b62c 100644 --- a/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh +++ b/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh @@ -32,7 +32,7 @@ __global__ void _BinaryElementWise( if (id < N) { CUDA_LONG lhs_index = (lhs_need_compute ? 0 : id); CUDA_LONG rhs_index = (rhs_need_compute ? 0 : id); - // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md + // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/main/docs/Broadcasting.md CUDA_LONG offset = id; #pragma unroll for (auto dim = 0; dim < fdm_output_strides.Capacity(); dim++) { @@ -188,7 +188,7 @@ void BinaryElementWiseNoBroadcastImpl( size_t count) { if (count == 0) // special case where there's a dim value of 0 in the output shape return; - + #ifdef USE_ROCM const int num_elements_per_thread = 2; const int num_threads_per_block = 512; diff --git a/onnxruntime/core/providers/cuda/cu_inc/common.cuh b/onnxruntime/core/providers/cuda/cu_inc/common.cuh index 7722ca61e6..1d3e43f386 100644 --- a/onnxruntime/core/providers/cuda/cu_inc/common.cuh +++ b/onnxruntime/core/providers/cuda/cu_inc/common.cuh @@ -75,10 +75,33 @@ __device__ __forceinline__ bool operator>=(const half& lh, const half& rh) { ret __device__ __forceinline__ bool operator<=(const half& lh, const half& rh) { return (float)lh <= (float)rh; } // support half2 arithmetic for cuda architecture < 5.3 -__device__ __forceinline__ half2 operator+(const half2& lh, const half2& rh) { half2 r; r.x = lh.x + rh.x; r.y = lh.y + rh.y; return r; } -__device__ __forceinline__ half2 operator-(const half2& lh, const half2& rh) { half2 r; r.x = lh.x - rh.x; r.y = lh.y - rh.y; return r; } -__device__ __forceinline__ half2 operator*(const half2& lh, const half2& rh) { half2 r; r.x = lh.x * rh.x; r.y = lh.y * rh.y; return r; } -__device__ __forceinline__ half2 operator/(const half2& lh, const half2& rh) { half2 r; r.x = lh.x / rh.x; r.y = lh.y / rh.y; return r; } +__device__ __forceinline__ half2 operator+(const half2& lh, const half2& rh) { + half2 r; + r.x = lh.x + rh.x; + r.y = lh.y + rh.y; + return r; +} + +__device__ __forceinline__ half2 operator-(const half2& lh, const half2& rh) { + half2 r; + r.x = lh.x - rh.x; + r.y = lh.y - rh.y; + return r; +} + +__device__ __forceinline__ half2 operator*(const half2& lh, const half2& rh) { + half2 r; + r.x = lh.x * rh.x; + r.y = lh.y * rh.y; + return r; +} + +__device__ __forceinline__ half2 operator/(const half2& lh, const half2& rh) { + half2 r; + r.x = lh.x / rh.x; + r.y = lh.y / rh.y; + return r; +} #endif /// Arithmetic for BFloat16 @@ -414,7 +437,7 @@ struct GridDim { }; // aligned vector generates vectorized load/store on CUDA -template +template struct alignas(sizeof(T) * vec_size) aligned_vector { T val[vec_size]; }; diff --git a/onnxruntime/core/providers/cuda/cuda_call.cc b/onnxruntime/core/providers/cuda/cuda_call.cc index 7c065d9e61..09e575328a 100644 --- a/onnxruntime/core/providers/cuda/cuda_call.cc +++ b/onnxruntime/core/providers/cuda/cuda_call.cc @@ -85,7 +85,8 @@ const char* CudaErrString(ncclResult_t e) { #endif template -bool CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { +std::conditional_t CudaCall( + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { if (retCode != successCode) { try { #ifdef _WIN32 @@ -111,36 +112,40 @@ bool CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRT libName, (int)retCode, CudaErrString(retCode), currentCudaDevice, hostname, exprString, msg); - if (THRW) { + if constexpr (THRW) { // throw an exception with the error info ORT_THROW(str); } else { LOGS_DEFAULT(ERROR) << str; + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, str); } } catch (const std::exception& e) { // catch, log, and rethrow since CUDA code sometimes hangs in destruction, so we'd never get to see the error - if (THRW) { + if constexpr (THRW) { ORT_THROW(e.what()); } else { LOGS_DEFAULT(ERROR) << e.what(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, e.what()); } } - return false; } - return true; + if constexpr (!THRW) { + return Status::OK(); + } } -template bool CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); -template bool CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); -template bool CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); -template bool CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); -template bool CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); -template bool CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); -template bool CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg); -template bool CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg); -template bool CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg); -template bool CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg); +template Status CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); +template void CudaCall(cudaError retCode, const char* exprString, const char* libName, cudaError successCode, const char* msg); +template Status CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); +template void CudaCall(cublasStatus_t retCode, const char* exprString, const char* libName, cublasStatus_t successCode, const char* msg); +template Status CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); +template void CudaCall(cudnnStatus_t retCode, const char* exprString, const char* libName, cudnnStatus_t successCode, const char* msg); +template Status CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg); +template void CudaCall(curandStatus_t retCode, const char* exprString, const char* libName, curandStatus_t successCode, const char* msg); +template Status CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg); +template void CudaCall(cufftResult retCode, const char* exprString, const char* libName, cufftResult successCode, const char* msg); #ifdef ORT_USE_NCCL -template bool CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); +template Status CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); +template void CudaCall(ncclResult_t retCode, const char* exprString, const char* libName, ncclResult_t successCode, const char* msg); #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_check_memory.cc b/onnxruntime/core/providers/cuda/cuda_check_memory.cc index 3dc2389fa3..547ecae598 100644 --- a/onnxruntime/core/providers/cuda/cuda_check_memory.cc +++ b/onnxruntime/core/providers/cuda/cuda_check_memory.cc @@ -7,9 +7,9 @@ namespace onnxruntime { void CheckIfMemoryOnCurrentGpuDevice(const void* ptr) { cudaPointerAttributes attrs; - CUDA_CALL(cudaPointerGetAttributes(&attrs, ptr)); + CUDA_CALL_THROW(cudaPointerGetAttributes(&attrs, ptr)); int current_device; - CUDA_CALL(cudaGetDevice(¤t_device)); + CUDA_CALL_THROW(cudaGetDevice(¤t_device)); ORT_ENFORCE(attrs.device == current_device, "Current CUDA device is ", current_device, " but the memory of pointer ", ptr, diff --git a/onnxruntime/core/providers/cuda/cuda_common.h b/onnxruntime/core/providers/cuda/cuda_common.h index 1d41b47536..7c7103a14d 100644 --- a/onnxruntime/core/providers/cuda/cuda_common.h +++ b/onnxruntime/core/providers/cuda/cuda_common.h @@ -14,40 +14,13 @@ namespace onnxruntime { namespace cuda { -#define CUDA_RETURN_IF_ERROR(expr) \ - ORT_RETURN_IF_ERROR(CUDA_CALL(expr) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CUDA error executing ", #expr)) - -#define CUBLAS_RETURN_IF_ERROR(expr) \ - ORT_RETURN_IF_ERROR(CUBLAS_CALL(expr) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CUBLAS error executing ", #expr)) - -#define CUSPARSE_RETURN_IF_ERROR(expr) \ - ORT_RETURN_IF_ERROR(CUSPARSE_CALL(expr) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CUSPARSE error executing ", #expr)) - -#define CURAND_RETURN_IF_ERROR(expr) \ - ORT_RETURN_IF_ERROR(CURAND_CALL(expr) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CURAND error executing ", #expr)) - -#define CUDNN_RETURN_IF_ERROR(expr) \ - ORT_RETURN_IF_ERROR(CUDNN_CALL(expr) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CUDNN error executing ", #expr)) - -#define CUDNN2_RETURN_IF_ERROR(expr, m) \ - ORT_RETURN_IF_ERROR(CUDNN_CALL2(expr, m) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CUDNN2 error executing ", #expr)) - -#define CUFFT_RETURN_IF_ERROR(expr) \ - ORT_RETURN_IF_ERROR(CUFFT_CALL(expr) \ - ? common::Status::OK() \ - : ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "CUFFT error executing ", #expr)) +#define CUDA_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR(CUDA_CALL(expr)) +#define CUBLAS_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR(CUBLAS_CALL(expr)) +#define CUSPARSE_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR(CUSPARSE_CALL(expr)) +#define CURAND_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR(CURAND_CALL(expr)) +#define CUDNN_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR(CUDNN_CALL(expr)) +#define CUDNN2_RETURN_IF_ERROR(expr, m) ORT_RETURN_IF_ERROR(CUDNN_CALL2(expr, m)) +#define CUFFT_RETURN_IF_ERROR(expr) ORT_RETURN_IF_ERROR(CUFFT_CALL(expr)) // Type mapping for MLFloat16 to half template diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc index 57e0f1aaf9..486adf7521 100755 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc @@ -127,7 +127,6 @@ AllocatorPtr CUDAExecutionProvider::CreateCudaAllocator(OrtDevice::DeviceId devi false); return CreateAllocator(default_memory_info); - } else { AllocatorCreationInfo default_memory_info( [](OrtDevice::DeviceId id) { @@ -150,6 +149,7 @@ CUDAExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId de stream_ = stream; CUBLAS_CALL_THROW(cublasCreate(&cublas_handle_)); + CUBLAS_CALL_THROW(cublasLtCreate(&cublas_lt_handle_)); CUBLAS_CALL_THROW(cublasSetStream(cublas_handle_, stream)); CUDNN_CALL_THROW(cudnnCreate(&cudnn_handle_)); @@ -164,20 +164,9 @@ CUDAExecutionProvider::PerThreadContext::PerThreadContext(OrtDevice::DeviceId de } CUDAExecutionProvider::PerThreadContext::~PerThreadContext() { - // dtor shouldn't throw. if something went wrong earlier (e.g. out of CUDA memory) the handles - // here may be bad, and the destroy calls can throw. - // https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Rc-dtor-noexcept - try { - CUBLAS_CALL(cublasDestroy(cublas_handle_)); - } catch (const std::exception& ex) { - LOGS_DEFAULT(ERROR) << "cublasDestroy threw:" << ex.what(); - } - - try { - CUDNN_CALL(cudnnDestroy(cudnn_handle_)); - } catch (const std::exception& ex) { - LOGS_DEFAULT(ERROR) << "cudnnDestroy threw:" << ex.what(); - } + ORT_IGNORE_RETURN_VALUE(CUBLAS_CALL(cublasDestroy(cublas_handle_))); + ORT_IGNORE_RETURN_VALUE(CUBLAS_CALL(cublasLtDestroy(cublas_lt_handle_))); + ORT_IGNORE_RETURN_VALUE(CUDNN_CALL(cudnnDestroy(cudnn_handle_))); } #if defined(CUDA_VERSION) && CUDA_VERSION >= 10000 @@ -238,22 +227,9 @@ CUDAExecutionProvider::CUDAExecutionProvider(const CUDAExecutionProviderInfo& in } CUDAExecutionProvider::~CUDAExecutionProvider() { - auto cpu_alloc = GetAllocator(DEFAULT_CPU_ALLOCATOR_DEVICE_ID, OrtMemTypeCPU); - { - std::lock_guard lock(deferred_release_cpu_ptr_mutex_); - auto it = deferred_release_cpu_ptr_.begin(); - while (it != deferred_release_cpu_ptr_.end()) { - auto& e = it->first; - auto& v = it->second; - if (v.recorded) - CUDA_CALL_THROW(cudaEventSynchronize(e)); - for (auto p : v.cpu_ptrs) { - cpu_alloc->Free(p); - } - CUDA_CALL_THROW(cudaEventDestroy(e)); - it = deferred_release_cpu_ptr_.erase(it); - } - } + // Prevent memory leak when people don't call + // OnRunStart and OnRunEnd when calling CudaKernel's. + ORT_IGNORE_RETURN_VALUE(EnqueueDeferredRelease()); // clean up thread local context caches { @@ -266,7 +242,7 @@ CUDAExecutionProvider::~CUDAExecutionProvider() { } if (!external_stream_ && stream_) { - CUDA_CALL(cudaStreamDestroy(stream_)); + ORT_IGNORE_RETURN_VALUE(CUDA_CALL(cudaStreamDestroy(stream_))); } } @@ -274,6 +250,16 @@ std::unique_ptr CUDAExecutionProvider::GetProfiler() { return std::make_unique(); } +// Suppressing warning "C26816: The pointer points to memory allocated on the stack." for +// CUDAExecutionProvider::GetPerThreadContext(). +// While CUDAExecutionProvider::GetPerThreadContext() does return the result of dereferencing a local +// std::shared_ptr, the underlying PerThreadContext is owned by the CUDA EP and is not local to this +// function. +#ifdef _WIN32 +#pragma warning(push) +#pragma warning(disable : 26816) +#endif + CUDAExecutionProvider::PerThreadContext& CUDAExecutionProvider::GetPerThreadContext() const { const auto& per_thread_context_cache = PerThreadContextCache(); @@ -312,6 +298,10 @@ CUDAExecutionProvider::PerThreadContext& CUDAExecutionProvider::GetPerThreadCont return *context; } +#ifdef _WIN32 +#pragma warning(pop) +#endif + void CUDAExecutionProvider::ReleasePerThreadContext() const { const auto& per_thread_context_cache = PerThreadContextCache(); @@ -349,43 +339,91 @@ void CUDAExecutionProvider::AddDeferredReleaseCPUPtr(void* p) { // when not running in InferenceSession (e.g. Test) // it's OK to not remember the deferred release ptr // as the actual memory will be cleaned in arena allocator dtor - auto current_deferred_release_event = GetPerThreadContext().GetCurrentDeferredReleaseEvent(); - if (current_deferred_release_event) { - std::lock_guard lock(deferred_release_cpu_ptr_mutex_); - auto iter = deferred_release_cpu_ptr_.find(current_deferred_release_event); - ORT_ENFORCE(iter != deferred_release_cpu_ptr_.end()); - iter->second.cpu_ptrs.push_back(p); + + // This function should only record pointers returned by + // AllocateBufferOnCPUPinned. + + std::lock_guard lock(deferred_release_mutex_); + void* stream = GetComputeStream(); + auto it = deferred_release_buffer_pool_.find(stream); + if (it != deferred_release_buffer_pool_.end()) { + it->second.push_back(p); + } else { + deferred_release_buffer_pool_[stream] = {p}; } } +struct CpuBuffersInfo { + // This struct stores the information needed + // to release CPU buffers allocated for GPU kernels. + // It's used to enqueue their release after + // associated GPU kernels in a CUDA stream. + + // This is a CPU allocator in CUDA EP. + // It must be the one used to allocate the + // following pointers. + AllocatorPtr allocator; + // buffers[i] is the i-th pointer added by + // AddDeferredReleaseCPUPtr for a specific + // CUDA stream. For example, this fields + // should contain all values in + // deferred_release_buffer_pool_[my_stream] + // when release my_stream's buffers. + void** buffers; + // CPU buffer buffers[i]. + // Number of buffer points in "buffers". + size_t n_buffers; +}; + +static void CUDART_CB ReleaseCpuBufferCallback(void* raw_info) { + auto info = reinterpret_cast(raw_info); + // Uncomment the following line to check if all previous stream + // operations are done correctly. + // checkCudaErrors(tmp->status); + for (size_t i = 0; i < info->n_buffers; ++i) { + info->allocator->Free(info->buffers[i]); + } + delete[] info->buffers; + delete info; +} + +Status CUDAExecutionProvider::EnqueueDeferredRelease() { + // Release CPU buffers allocated for CUDA kernels (type: CudaKernel). + // They have to be released outside CUDA kernels because they must be alive + // during asynchronous GPU computation even after the CPU part (e.g, + // CudaKernel::ComputeInternal) already return. + std::lock_guard lock(deferred_release_mutex_); + for (auto it = deferred_release_buffer_pool_.begin(); it != deferred_release_buffer_pool_.end(); ++it) { + // it->first: a CUDA stream. + // it->second: CPU buffers associated with kernels running on it->first. + // This iteration enqueues a callback to release all buffers + // in it->second on it->first. + + auto stream = static_cast(it->first); + auto& buffers = it->second; + // Allocate a heap object to extend the lifetime of allocator and buffer pointers + // until the end of callback (aka ReleaseCpuBufferCallback). + auto cpu_buffers_info = new CpuBuffersInfo; + // This allocator must be the same to the allocator + // used in AllocateBufferOnCPUPinned. + cpu_buffers_info->allocator = GetAllocator(DEFAULT_CPU_ALLOCATOR_DEVICE_ID, OrtMemTypeCPU); + cpu_buffers_info->buffers = new void*[buffers.size()]; + for (size_t i = 0; i < buffers.size(); ++i) { + cpu_buffers_info->buffers[i] = buffers.at(i); + } + cpu_buffers_info->n_buffers = buffers.size(); + CUDA_RETURN_IF_ERROR(cudaLaunchHostFunc(stream, ReleaseCpuBufferCallback, cpu_buffers_info)); + } + // All buffers are scheduled for release. + // Let's clear releated information so that + // those buffers won't be released twice. + deferred_release_buffer_pool_.clear(); + return Status::OK(); +} + Status CUDAExecutionProvider::OnRunStart() { // always set CUDA device when session::Run() in case it runs in a worker thread CUDA_RETURN_IF_ERROR(cudaSetDevice(GetDeviceId())); - auto cpu_alloc = GetAllocator(0, OrtMemTypeCPU); - // check if cudaEvents has passed for deferred release - // note that we need to take a mutex in case of multi-threaded Run() - std::lock_guard lock(deferred_release_cpu_ptr_mutex_); - auto it = deferred_release_cpu_ptr_.begin(); - while (it != deferred_release_cpu_ptr_.end()) { - auto& e = it->first; - auto& v = it->second; - // note that cudaEventQuery returns cudaSucess before first cudaEventRecord - if (v.recorded && cudaSuccess == cudaEventQuery(e)) { - for (auto p : v.cpu_ptrs) { - cpu_alloc->Free(p); - } - cudaEvent_t expired_event = it->first; - it = deferred_release_cpu_ptr_.erase(it); - CUDA_RETURN_IF_ERROR(cudaEventDestroy(expired_event)); - } else { - ++it; - } - } - - auto& current_deferred_release_event = GetPerThreadContext().GetCurrentDeferredReleaseEvent(); - CUDA_RETURN_IF_ERROR(cudaEventCreate(¤t_deferred_release_event, cudaEventDisableTiming)); - deferred_release_cpu_ptr_.emplace(current_deferred_release_event, DeferredReleaseCPUPtrs()); - if (IsGraphCaptureEnabled() && GetPerThreadContext().IsGraphCaptureAllowed() && !GetPerThreadContext().IsGraphCaptured()) { LOGS_DEFAULT(INFO) << "Capturing the cuda graph for this model"; GetPerThreadContext().CaptureBegin(); @@ -404,21 +442,29 @@ Status CUDAExecutionProvider::OnRunEnd(bool sync_stream) { GetPerThreadContext().IncrementRegularRunCountBeforeGraphCapture(); } } - // record deferred release event on default stream, and release per_thread_context - auto current_deferred_release_event = GetPerThreadContext().GetCurrentDeferredReleaseEvent(); - CUDA_RETURN_IF_ERROR(cudaEventRecord(current_deferred_release_event, static_cast(GetComputeStream()))); + + // Enqueue deferred CPU memory release on related streams. + // This will release all deferred-release CPU buffers allocated + // before calling OnRunEnd. + ORT_RETURN_IF_ERROR(EnqueueDeferredRelease()); + if (sync_stream) { CUDA_RETURN_IF_ERROR(cudaStreamSynchronize(static_cast(GetComputeStream()))); } - // If cuda graph is enabled, the per thread context will not be released - // because the per thread cuda graph needs to be maintained and replayed for - // the next run. - if (!IsGraphCaptureEnabled()) { + // The reason of !IsGraphCaptureEnabled(): + // If cuda graph is enabled, the per thread context will not be released + // because the per thread cuda graph needs to be maintained and replayed for + // the next run. + // The reason of PerThreadContextCache()->find(this) != PerThreadContextCache()->end(): + // In extreme cases (e.g., 1-op graph and that op fallbacks to CPU), + // PerThreadContext won't be created and there isbe nothing to + // release. This didn't happen before because we always call + // GetPerThreadContext in OnRunStart. + if (!IsGraphCaptureEnabled() && + PerThreadContextCache()->find(this) != PerThreadContextCache()->end()) { ReleasePerThreadContext(); } - std::lock_guard lock(deferred_release_cpu_ptr_mutex_); - deferred_release_cpu_ptr_[current_deferred_release_event].recorded = true; return Status::OK(); } diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h index cb126f203b..f7031368b6 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h @@ -45,6 +45,10 @@ class CUDAExecutionProvider : public IExecutionProvider { return GetPerThreadContext().CublasHandle(); } + cublasLtHandle_t PerThreadCublasLtHandle() { + return GetPerThreadContext().CublasLtHandle(); + } + cudnnHandle_t PerThreadCudnnHandle() { return GetPerThreadContext().CudnnHandle(); } @@ -54,7 +58,23 @@ class CUDAExecutionProvider : public IExecutionProvider { return GetPerThreadContext().template GetConstOnes(count); } + // Add CPU buffer to a buffer pool. + // They can and only can be released + // by calling EuqueueDeferredRelease. + // A common pattern is + // 1. auto buffer = AllocateBufferOnCPUPinned(128); + // 2. Some GPU kernel calls on GPU stream from GetComputeStream. + // 3. Call AddDeferredReleaseCPUPtr(buffer.release()); + // 4. Call EnqueueDeferredRelease(); + // so that the allocated "buffer" in (1) will be released + // only after all GPU kernels in (2) are finished. + // (4) is done in OnRunEnd, so the user doesn't need to call + // it in most cases. void AddDeferredReleaseCPUPtr(void* p); + // Release all buffers added by + // AddDeferredReleaseCPUPtr using + // GPU callback (so it's async). + Status EnqueueDeferredRelease(); template IAllocatorUniquePtr GetScratchBuffer(size_t count_or_bytes) const { @@ -108,13 +128,17 @@ class CUDAExecutionProvider : public IExecutionProvider { bool external_stream_ = false; cudaStream_t stream_ = nullptr; - struct DeferredReleaseCPUPtrs { - bool recorded = false; - std::vector cpu_ptrs; - }; - - std::unordered_map deferred_release_cpu_ptr_; - OrtMutex deferred_release_cpu_ptr_mutex_; + // deferred_release_buffer_pool_[my_stream] store all CPU buffers associated with + // CUDA kernels running on my_stream (type: cudaStream_t). + // Buffers' release is enqueued as a CUDA callback onto the associated stream (aka + // stream returned by GetComputeStream when calling AddDeferredReleaseCPUPtr) in OnRunEnd. + // Those are pointers allocated by AllocateBufferOnCPUPinned and should be released + // by CPU Allocator's Free function. + std::unordered_map> deferred_release_buffer_pool_; + // To add a pointer to deferred_release_buffer_pool_, we need a mutex because + // different threads may create CPU buffers at the same time. Releasing + // buffers also needs this mutex. + OrtMutex deferred_release_mutex_; class PerThreadContext final { public: @@ -130,8 +154,8 @@ class CUDAExecutionProvider : public IExecutionProvider { return cudnn_handle_; } - cudaEvent_t& GetCurrentDeferredReleaseEvent() { - return current_deferred_release_event_; + cublasLtHandle_t CublasLtHandle() const { + return cublas_lt_handle_; } template @@ -166,23 +190,19 @@ class CUDAExecutionProvider : public IExecutionProvider { } #if defined(CUDA_VERSION) && CUDA_VERSION >= 10000 - bool IsGraphCaptureAllowed() const; - void CaptureBegin(); - void CaptureEnd(); - bool IsGraphCaptured() const; - Status ReplayGraph(); - void IncrementRegularRunCountBeforeGraphCapture(); + bool IsGraphCaptureAllowed() const; + void CaptureBegin(); + void CaptureEnd(); + bool IsGraphCaptured() const; + Status ReplayGraph(); + void IncrementRegularRunCountBeforeGraphCapture(); #endif private: cudaStream_t stream_ = nullptr; cublasHandle_t cublas_handle_ = nullptr; cudnnHandle_t cudnn_handle_ = nullptr; - - // deferred release for temporary CPU pinned memory used in cudaMemcpyAsync - // note that cudaEvent will be assigned at OnRunEnd() when PerThreadContext destory - // so the ownership is passed to deferred_release_cpu_ptr_ - cudaEvent_t current_deferred_release_event_ = nullptr; + cublasLtHandle_t cublas_lt_handle_ = nullptr; std::unique_ptr> constant_ones_float_; std::unique_ptr> constant_ones_double_; @@ -197,10 +217,9 @@ class CUDAExecutionProvider : public IExecutionProvider { CUDAGraph cuda_graph_; bool is_graph_captured_ = false; int regular_run_count_before_graph_capture_ = 0; - const int min_num_runs_before_cuda_graph_capture_ = 1; // required min regular runs before graph capture for the necessary memory allocations. + const int min_num_runs_before_cuda_graph_capture_ = 1; // required min regular runs before graph capture for the necessary memory allocations. #endif - }; using PerThreadContextMap = std::unordered_map>; @@ -245,4 +264,4 @@ class CUDAExecutionProvider : public IExecutionProvider { void ReleasePerThreadContext() const; }; -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc index 948914158f..257189318b 100644 --- a/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc +++ b/onnxruntime/core/providers/cuda/cuda_execution_provider_info.cc @@ -52,9 +52,7 @@ CUDAExecutionProviderInfo CUDAExecutionProviderInfo::FromProviderOptions(const P [&info](const std::string& value_str) -> Status { ORT_RETURN_IF_ERROR(ParseStringWithClassicLocale(value_str, info.device_id)); int num_devices{}; - ORT_RETURN_IF_NOT( - CUDA_CALL(cudaGetDeviceCount(&num_devices)), - "cudaGetDeviceCount() failed."); + CUDA_RETURN_IF_ERROR(cudaGetDeviceCount(&num_devices)); ORT_RETURN_IF_NOT( 0 <= info.device_id && info.device_id < num_devices, "Invalid device ID: ", info.device_id, diff --git a/onnxruntime/core/providers/cuda/cuda_kernel.h b/onnxruntime/core/providers/cuda/cuda_kernel.h index 7ea2a07ae0..2ce72d7112 100644 --- a/onnxruntime/core/providers/cuda/cuda_kernel.h +++ b/onnxruntime/core/providers/cuda/cuda_kernel.h @@ -135,6 +135,10 @@ class CudaKernel : public OpKernel { return provider_->PerThreadCublasHandle(); } + inline cublasLtHandle_t CublasLtHandle() const { + return provider_->PerThreadCublasLtHandle(); + } + inline cudnnHandle_t CudnnHandle() const { return provider_->PerThreadCudnnHandle(); } diff --git a/onnxruntime/core/providers/cuda/cuda_pch.h b/onnxruntime/core/providers/cuda/cuda_pch.h index a671b91fb1..bdb0e78d77 100644 --- a/onnxruntime/core/providers/cuda/cuda_pch.h +++ b/onnxruntime/core/providers/cuda/cuda_pch.h @@ -15,6 +15,7 @@ #include #include #include +#include #ifdef ORT_USE_NCCL #include diff --git a/onnxruntime/core/providers/cuda/cuda_profiler.cc b/onnxruntime/core/providers/cuda/cuda_profiler.cc index adf771fb23..817816f43b 100644 --- a/onnxruntime/core/providers/cuda/cuda_profiler.cc +++ b/onnxruntime/core/providers/cuda/cuda_profiler.cc @@ -7,6 +7,8 @@ #include #include +#include "core/common/profiler_common.h" + namespace onnxruntime { namespace profiling { @@ -126,7 +128,7 @@ void CudaProfiler::EndProfiling(TimePoint start_time, Events& events) { {"block_y", std::to_string(stat.block_y_)}, {"block_z", std::to_string(stat.block_z_)}}; EventRecord event{ - KEVENT, -1, -1, stat.name_, DUR(profiling_start, stat.start_), DUR(stat.start_, stat.stop_), {args.begin(), args.end()}}; + KEVENT, -1, -1, demangle(stat.name_), DUR(profiling_start, stat.start_), DUR(stat.start_, stat.stop_), {args.begin(), args.end()}}; auto ts = id_map[stat.correlation_id]; if (event_map.find(ts) == event_map.end()) { event_map.insert({ts, {event}}); diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc index 33bf45a447..52985d4034 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.cc @@ -106,26 +106,26 @@ struct ProviderInfo_CUDA_Impl : ProviderInfo_CUDA { return cuda::Impl_Cast(static_cast(stream), input_data, output_data, count); } - bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } - bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } + Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { return CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } + void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) override { CudaCall(cudaError(retCode), exprString, libName, cudaError(successCode), msg); } void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) override { ORT_ENFORCE(dst_location.device.Type() == OrtDevice::CPU); // Current CUDA device. int device; - CUDA_CALL(cudaGetDevice(&device)); + CUDA_CALL_THROW(cudaGetDevice(&device)); if (device != src_location.id) { // Need to switch to the allocating device. - CUDA_CALL(cudaSetDevice(src_location.id)); + CUDA_CALL_THROW(cudaSetDevice(src_location.id)); // Copy from GPU to CPU. - CUDA_CALL(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); + CUDA_CALL_THROW(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); // Switch back to current device. - CUDA_CALL(cudaSetDevice(device)); + CUDA_CALL_THROW(cudaSetDevice(device)); } else { // Copy from GPU to CPU. - CUDA_CALL(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); + CUDA_CALL_THROW(cudaMemcpy(dst_ptr, src_ptr, size, cudaMemcpyDeviceToHost)); } } diff --git a/onnxruntime/core/providers/cuda/cuda_provider_factory.h b/onnxruntime/core/providers/cuda/cuda_provider_factory.h index 927ffd4107..bc04270b88 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_factory.h +++ b/onnxruntime/core/providers/cuda/cuda_provider_factory.h @@ -32,8 +32,8 @@ struct ProviderInfo_CUDA { virtual void cuda__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) = 0; virtual void cuda__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) = 0; - virtual bool CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; - virtual bool CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual Status CudaCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; + virtual void CudaCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg) = 0; virtual void CopyGpuToCpu(void* dst_ptr, const void* src_ptr, const size_t size, const OrtMemoryInfo& dst_location, const OrtMemoryInfo& src_location) = 0; virtual void cudaMemcpy_HostToDevice(void* dst, const void* src, size_t count) = 0; diff --git a/onnxruntime/core/providers/cuda/generator/range.cc b/onnxruntime/core/providers/cuda/generator/range.cc index 651ed6cf1f..56189875df 100644 --- a/onnxruntime/core/providers/cuda/generator/range.cc +++ b/onnxruntime/core/providers/cuda/generator/range.cc @@ -71,10 +71,7 @@ static Status ComputeRange(cudaStream_t stream, OpKernelContext* ctx) { T* y = ctx->Output(0, shape)->MutableData(); if (count > 0) { - if (!RangeImpl(stream, start, delta, count, y)) { - CUDA_CALL(cudaGetLastError()); - return Status(common::ONNXRUNTIME, common::FAIL); - } + ORT_RETURN_IF_ERROR(RangeImpl(stream, start, delta, count, y)); } return Status::OK(); diff --git a/onnxruntime/core/providers/cuda/generator/range_impl.cu b/onnxruntime/core/providers/cuda/generator/range_impl.cu index ede8146478..6203dd4aac 100644 --- a/onnxruntime/core/providers/cuda/generator/range_impl.cu +++ b/onnxruntime/core/providers/cuda/generator/range_impl.cu @@ -22,15 +22,15 @@ __global__ void RangeKernel(const T start, const T delta, const int count, T* ou } template -bool RangeImpl(cudaStream_t stream, const T start, const T delta, const int count, T* output) { +Status RangeImpl(cudaStream_t stream, const T start, const T delta, const int count, T* output) { constexpr int block_size = 256; int grid_size = (count + block_size - 1) / block_size; RangeKernel<<>>(start, delta, count, output); - return CUDA_CALL(cudaPeekAtLastError()); + return CUDA_CALL(cudaGetLastError()); } #define SPECIALIZED_IMPL(T) \ - template bool RangeImpl(cudaStream_t stream, const T start, const T delta, const int count, T* output); + template Status RangeImpl(cudaStream_t stream, const T start, const T delta, const int count, T* output); SPECIALIZED_IMPL(int16_t) SPECIALIZED_IMPL(int32_t) diff --git a/onnxruntime/core/providers/cuda/generator/range_impl.h b/onnxruntime/core/providers/cuda/generator/range_impl.h index 608c65223a..d7585b941a 100644 --- a/onnxruntime/core/providers/cuda/generator/range_impl.h +++ b/onnxruntime/core/providers/cuda/generator/range_impl.h @@ -9,7 +9,7 @@ namespace cuda { template -bool RangeImpl(cudaStream_t stream, const T start, const T delta, const int count, T* output); +Status RangeImpl(cudaStream_t stream, const T start, const T delta, const int count, T* output); } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/gpu_data_transfer.cc b/onnxruntime/core/providers/cuda/gpu_data_transfer.cc index a8a7d2eb60..759b67d118 100644 --- a/onnxruntime/core/providers/cuda/gpu_data_transfer.cc +++ b/onnxruntime/core/providers/cuda/gpu_data_transfer.cc @@ -25,10 +25,10 @@ GPUDataTransfer::GPUDataTransfer(cudaStream_t stream, bool do_copy_in_default_st GPUDataTransfer::~GPUDataTransfer() { if (!do_copy_in_default_stream_ && streams_[kCudaStreamCopyIn] != nullptr) { - CUDA_CALL(cudaStreamDestroy(streams_[kCudaStreamCopyIn])); + ORT_IGNORE_RETURN_VALUE(CUDA_CALL(cudaStreamDestroy(streams_[kCudaStreamCopyIn]))); } if (!do_copy_in_default_stream_ && streams_[kCudaStreamCopyOut] != nullptr) { - CUDA_CALL(cudaStreamDestroy(streams_[kCudaStreamCopyOut])); + ORT_IGNORE_RETURN_VALUE(CUDA_CALL(cudaStreamDestroy(streams_[kCudaStreamCopyOut]))); } } diff --git a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h index fcab87c193..c07324a8ac 100644 --- a/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h +++ b/onnxruntime/core/providers/cuda/math/binary_elementwise_ops.h @@ -57,7 +57,7 @@ struct BinaryElementwisePreparation { [&C](int64_t dim) { if (dim != 1) C = dim; return (dim != 1); })) { int32_t dim_C = gsl::narrow_cast(std::find(rhs_dims.begin(), rhs_dims.end(), C) - rhs_dims.begin() + output_shape.NumDimensions() - rhs_shape.NumDimensions()); int64_t N = output_shape.SizeToDimension(dim_C); - int64_t H = (dim_C < out_rank - 1 ? output_shape.SizeFromDimension(dim_C + 1) : 1); + int64_t H = (dim_C < out_rank - 1 ? output_shape.SizeFromDimension(static_cast(dim_C) + 1) : 1); std::vector new_output_dims; if (N == 1) { diff --git a/onnxruntime/core/providers/cuda/math/matmul_integer.cu b/onnxruntime/core/providers/cuda/math/matmul_integer.cu index f6a9d6488b..757e39dc44 100644 --- a/onnxruntime/core/providers/cuda/math/matmul_integer.cu +++ b/onnxruntime/core/providers/cuda/math/matmul_integer.cu @@ -34,7 +34,7 @@ Status ReduceRowSumOnMatrixA(cudaStream_t stream, const int8_t* matrix, int32_t* static_cast(helper.K())); } - return CUDA_CALL(cudaPeekAtLastError()) ? Status::OK() : Status(common::ONNXRUNTIME, common::FAIL); + return CUDA_CALL(cudaGetLastError()); } template @@ -63,7 +63,7 @@ Status ReduceColSumOnMatrixB(cudaStream_t stream, const int8_t* matrix, int32_t* static_cast(helper.N())); } - return CUDA_CALL(cudaPeekAtLastError()) ? Status::OK() : Status(common::ONNXRUNTIME, common::FAIL); + return CUDA_CALL(cudaGetLastError()); } __global__ void ComputeOffsetOfMatrixAB(const int32_t* row_sum, @@ -124,7 +124,7 @@ Status OffsetOutput(cudaStream_t stream, } } - return CUDA_CALL(cudaPeekAtLastError()) ? Status::OK() : Status(common::ONNXRUNTIME, common::FAIL); + return CUDA_CALL(cudaGetLastError()); } } // namespace cuda diff --git a/onnxruntime/core/providers/cuda/math/softmax.cc b/onnxruntime/core/providers/cuda/math/softmax.cc index 24d2515f5a..905df4ec1a 100644 --- a/onnxruntime/core/providers/cuda/math/softmax.cc +++ b/onnxruntime/core/providers/cuda/math/softmax.cc @@ -116,8 +116,8 @@ Status Softmax::ComputeInternal(OpKernelContext* ctx) const { std::vector permutation(rank); // The "semantic" meaning of axis has changed in opset-13. - // Please compare: https://github.com/onnx/onnx/blob/master/docs/Operators.md#Softmax - // with https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Softmax-11 for detailed explanations + // Please compare: https://github.com/onnx/onnx/blob/main/docs/Operators.md#Softmax + // with https://github.com/onnx/onnx/blob/main/docs/Changelog.md#Softmax-11 for detailed explanations // To account for the opset-13 behavior, our plan will be to transpose the "axis" dim to the innermost dim // and perform softmax and then reverse the transpose. We can skip the transposing aspect if the axis is already // the innermost dim diff --git a/onnxruntime/core/providers/cuda/math/softmax_common.cc b/onnxruntime/core/providers/cuda/math/softmax_common.cc new file mode 100644 index 0000000000..9520701700 --- /dev/null +++ b/onnxruntime/core/providers/cuda/math/softmax_common.cc @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/math/softmax_common.h" + +namespace onnxruntime { +namespace cuda { + +Status SoftmaxForward(cudnnHandle_t cudnn_handle, const void* alpha, const cudnnTensorDescriptor_t input_tensor, + const void* input_data, const void* beta, const cudnnTensorDescriptor_t output_tensor, + void* output_data) { + CUDNN_RETURN_IF_ERROR(cudnnSoftmaxForward(cudnn_handle, CUDNN_SOFTMAX_ACCURATE, CUDNN_SOFTMAX_MODE_INSTANCE, alpha, + input_tensor, input_data, beta, output_tensor, output_data)); + return Status::OK(); +} + +Status SoftmaxBackward(cudnnHandle_t cudnn_handle, bool is_log_softmax, const void* alpha, + const cudnnTensorDescriptor_t input_tensor, const void* output_data, + const void* output_grad_data, const void* beta, const cudnnTensorDescriptor_t output_tensor, + void* input_grad_data) { + CUDNN_RETURN_IF_ERROR(cudnnSoftmaxBackward(cudnn_handle, is_log_softmax ? CUDNN_SOFTMAX_LOG : CUDNN_SOFTMAX_ACCURATE, + CUDNN_SOFTMAX_MODE_INSTANCE, alpha, input_tensor, output_data, + input_tensor, output_grad_data, beta, output_tensor, input_grad_data)); + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/shared_inc/softmax.h b/onnxruntime/core/providers/cuda/math/softmax_common.h similarity index 56% rename from onnxruntime/core/providers/cuda/shared_inc/softmax.h rename to onnxruntime/core/providers/cuda/math/softmax_common.h index 0704a563af..643a1c283e 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/softmax.h +++ b/onnxruntime/core/providers/cuda/math/softmax_common.h @@ -11,11 +11,12 @@ namespace cuda { Status SoftmaxForward(cudnnHandle_t cudnn_handle, const void* alpha, const cudnnTensorDescriptor_t input_tensor, const void* input_data, const void* beta, const cudnnTensorDescriptor_t output_tensor, - void* output_data) { - CUDNN_RETURN_IF_ERROR(cudnnSoftmaxForward(cudnn_handle, CUDNN_SOFTMAX_ACCURATE, CUDNN_SOFTMAX_MODE_INSTANCE, alpha, - input_tensor, input_data, beta, output_tensor, output_data)); - return Status::OK(); -} + void* output_data); + +Status SoftmaxBackward(cudnnHandle_t cudnn_handle, bool is_log_softmax, const void* alpha, + const cudnnTensorDescriptor_t input_tensor, const void* output_data, + const void* output_grad_data, const void* beta, const cudnnTensorDescriptor_t output_tensor, + void* input_grad_data); } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc b/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc index 3404dd9c95..419743ecf7 100644 --- a/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc +++ b/onnxruntime/core/providers/cuda/math/variadic_elementwise_ops.cc @@ -235,46 +235,37 @@ using MinOp = VariadicElementwiseOp; - -const DeleteOnUnloadPtr> k_uzilhfd_datatypes = new std::vector( - BuildKernelDefConstraints()); -const DeleteOnUnloadPtr> k_hfd_datatypes = - new std::vector(BuildKernelDefConstraints()); - } // namespace // kernel registration -#define REGISTER_KERNEL(name, impl_class, version, datatypes) \ - ONNX_OPERATOR_KERNEL_EX( \ - name, \ - kOnnxDomain, \ - version, \ - kCudaExecutionProvider, \ - (*KernelDefBuilder::Create()).TypeConstraint("T", datatypes), \ - impl_class) +#define REGISTER_KERNEL(name, impl_class, version, datatypes) \ + ONNX_OPERATOR_KERNEL_EX(name, kOnnxDomain, version, kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", BuildKernelDefConstraints()), \ + impl_class) #define REGISTER_VERSIONED_KERNEL(name, impl_class, start_version, end_version, datatypes) \ ONNX_OPERATOR_VERSIONED_KERNEL_EX( \ - name, \ - kOnnxDomain, \ - start_version, end_version, \ - kCudaExecutionProvider, \ - (*KernelDefBuilder::Create()).TypeConstraint("T", datatypes), \ - impl_class) + name, kOnnxDomain, start_version, end_version, kCudaExecutionProvider, \ + (*KernelDefBuilder::Create()).TypeConstraint("T", BuildKernelDefConstraints()), impl_class) -REGISTER_KERNEL(Sum, SumOp, 13, *k_hfd_datatypes) -REGISTER_VERSIONED_KERNEL(Sum, SumOp, 8, 12, *k_hfd_datatypes) -REGISTER_VERSIONED_KERNEL(Sum, SumOp, 6, 7, *k_hfd_datatypes) +#define UZILHFD_TYPES uint32_t, uint64_t, int32_t, int64_t, MLFloat16, float, double, BFloat16 +#define HFD_TYPES MLFloat16, float, double, BFloat16 -REGISTER_KERNEL(Min, MinOp, 13, *k_uzilhfd_datatypes) -REGISTER_VERSIONED_KERNEL(Min, MinOp, 12, 12, *k_uzilhfd_datatypes) -REGISTER_VERSIONED_KERNEL(Min, MinOp, 6, 11, *k_hfd_datatypes) +REGISTER_KERNEL(Sum, SumOp, 13, HFD_TYPES) +REGISTER_VERSIONED_KERNEL(Sum, SumOp, 8, 12, HFD_TYPES) +REGISTER_VERSIONED_KERNEL(Sum, SumOp, 6, 7, HFD_TYPES) -REGISTER_KERNEL(Max, MaxOp, 13, *k_uzilhfd_datatypes) -REGISTER_VERSIONED_KERNEL(Max, MaxOp, 12, 12, *k_uzilhfd_datatypes) -REGISTER_VERSIONED_KERNEL(Max, MaxOp, 6, 11, *k_hfd_datatypes) +REGISTER_KERNEL(Min, MinOp, 13, UZILHFD_TYPES) +REGISTER_VERSIONED_KERNEL(Min, MinOp, 12, 12, UZILHFD_TYPES) +REGISTER_VERSIONED_KERNEL(Min, MinOp, 6, 11, HFD_TYPES) +REGISTER_KERNEL(Max, MaxOp, 13, UZILHFD_TYPES) +REGISTER_VERSIONED_KERNEL(Max, MaxOp, 12, 12, UZILHFD_TYPES) +REGISTER_VERSIONED_KERNEL(Max, MaxOp, 6, 11, HFD_TYPES) + +#undef HFD_TYPES +#undef UZILHFD_TYPES #undef REGISTER_VERSIONED_KERNEL #undef REGISTER_KERNEL diff --git a/onnxruntime/core/providers/cuda/nn/conv.cc b/onnxruntime/core/providers/cuda/nn/conv.cc index fd0d15640f..8e2983dd40 100644 --- a/onnxruntime/core/providers/cuda/nn/conv.cc +++ b/onnxruntime/core/providers/cuda/nn/conv.cc @@ -87,7 +87,7 @@ Status SliceOutUnwantedOutputSection(cudaStream_t stream, } template -Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const { +Status Conv::UpdateState(OpKernelContext* context) const { //set X const Tensor* X = context->Input(0); const TensorShape& x_shape = X->Shape(); @@ -109,8 +109,7 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const //set Z if (context->InputCount() >= 4) { const Tensor* Z = context->Input(3); - ORT_RETURN_IF_ERROR(s_.z_tensor.Set(Z->Shape().GetDims(), CudnnTensor::GetDataType())); - s_.z_data = reinterpret_cast(Z->Data()); + s_.z_data = reinterpret_cast(Z->template Data()); } else { s_.z_data = nullptr; } @@ -237,22 +236,43 @@ Status Conv::UpdateState(OpKernelContext* context, bool bias_expected) const if (context->InputCount() >= 3) { const Tensor* B = context->Input(2); const auto& b_shape = B->Shape(); - ORT_RETURN_IF_NOT(b_shape.NumDimensions() == 1, "bias should be 1D"); - TensorShapeVector b_dims(2 + kernel_shape.size(), 1); - b_dims[1] = b_shape[0]; - ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); - //s_.b_data = reinterpret_cast(B->Data()); - } else if (bias_expected) { - TensorShapeVector b_dims(2 + kernel_shape.size(), 1); - b_dims[1] = w_dims[0]; - auto malloc_size = b_dims[1] * sizeof(CudaT); - ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); - if (s_.b_zero) { - CUDA_CALL_THROW(cudaFree(s_.b_zero)); - s_.b_zero = nullptr; + if (b_shape.NumDimensions() == 1) { + TensorShapeVector b_dims(2 + kernel_shape.size(), 1); + b_dims[1] = b_shape[0]; + ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_dims, CudnnTensor::GetDataType())); + } else { + const auto& y_rank = y_dims_cudnn.size(); + const auto& b_rank = b_shape.GetDims().size(); + ORT_RETURN_IF_NOT(b_rank <= y_rank, "rank of B is ", b_rank, ", which is bigger than the rank of Y - ", y_rank); + if (b_rank == y_rank) { + ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_shape.GetDims(), CudnnTensor::GetDataType())); + } else { + TensorShapeVector b_extended_dims = b_shape.AsShapeVector(); + for (auto i = b_rank; i < y_rank; ++i) { + ORT_RETURN_IF_NOT(y_dims_cudnn[i] == 1, "dim ", i, " of Y is ", y_dims_cudnn[i], ", cannot apply it to that dim of B"); + b_extended_dims.push_back(1); + } + ORT_RETURN_IF_ERROR(s_.b_tensor.Set(b_extended_dims, CudnnTensor::GetDataType())); + } + } + } + + if (context->InputCount() >= 4) { + const Tensor* Z = context->Input(3); + const auto& z_shape = Z->Shape(); + const auto& z_rank = z_shape.GetDims().size(); + const auto& y_rank = y_dims_cudnn.size(); + ORT_RETURN_IF_NOT(z_rank <= y_rank, "rank of Z is ", z_rank, ", which is bigger than the rank of Y - ", y_rank); + if (z_rank == y_rank) { + ORT_RETURN_IF_ERROR(s_.z_tensor.Set(z_shape.GetDims(), CudnnTensor::GetDataType())); + } else { + TensorShapeVector z_extended_dims = z_shape.AsShapeVector(); + for (auto i = z_rank; i < y_rank; ++i) { + ORT_RETURN_IF_NOT(y_dims_cudnn[i] == 1, "dim ", i, " of Y is ", y_dims_cudnn[i], ", cannot apply it to that dim of Z"); + z_extended_dims.push_back(1); + } + ORT_RETURN_IF_ERROR(s_.z_tensor.Set(z_extended_dims, CudnnTensor::GetDataType())); } - CUDA_CALL_THROW(cudaMalloc(&s_.b_zero, malloc_size)); - CUDA_CALL_THROW(cudaMemsetAsync(s_.b_zero, 0, malloc_size, Stream())); } if (!s_.cached_benchmark_results.contains(x_dims_cudnn)) { diff --git a/onnxruntime/core/providers/cuda/nn/conv.h b/onnxruntime/core/providers/cuda/nn/conv.h index 135b189d4b..8227852552 100644 --- a/onnxruntime/core/providers/cuda/nn/conv.h +++ b/onnxruntime/core/providers/cuda/nn/conv.h @@ -141,7 +141,6 @@ struct CudnnConvState { const void* w_data = nullptr; CudnnTensor b_tensor; const void* b_data = nullptr; - void* b_zero = nullptr; CudnnTensor y_tensor; Tensor* Y = nullptr; void* y_data = nullptr; @@ -166,13 +165,6 @@ struct CudnnConvState { // note that conv objects are shared between execution frames, and a lock is needed to avoid multi-thread racing OrtMutex mutex; IAllocatorUniquePtr memory_for_cudnn_conv_results; - - ~CudnnConvState() { - if (b_zero) { - CUDA_CALL_THROW(cudaFree(b_zero)); - b_zero = nullptr; - } - } }; enum : size_t { @@ -197,7 +189,7 @@ class Conv : public CudaKernel { return GetScratchBuffer(s_.workspace_bytes); } - Status UpdateState(OpKernelContext* context, bool bias_expected = false) const; + Status UpdateState(OpKernelContext* context) const; ConvAttributes conv_attrs_; mutable CudnnConvState s_; constexpr static auto kDefaultConvAlgo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM; diff --git a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc index 20a787b475..0dbb31b0f4 100644 --- a/onnxruntime/core/providers/cuda/nn/conv_transpose.cc +++ b/onnxruntime/core/providers/cuda/nn/conv_transpose.cc @@ -3,6 +3,15 @@ #include "conv_transpose.h" +// To suppress FP static analyzer warnings: +// https://msdata.visualstudio.com/Vienna/_workitems/edit/1944928 and +// https://msdata.visualstudio.com/Vienna/_workitems/edit/1944950 +#ifdef _WIN32 +#pragma warning(push) +#pragma warning(disable : 26110) +#pragma warning(disable : 26117) +#endif + namespace onnxruntime { namespace cuda { @@ -206,3 +215,7 @@ Status ConvTranspose::DoConvTranspose(OpKernelContext* context, bool dynamic_ } // namespace cuda } // namespace onnxruntime + +#ifdef _WIN32 +#pragma warning(pop) +#endif diff --git a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc index 70391a8a00..0cc021cdd2 100644 --- a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc +++ b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.cc @@ -249,7 +249,7 @@ Status CudnnRnnBase::ComputeInternal(OpKernelContext* ctx) const { size_t workspace_bytes; CUDNN_RETURN_IF_ERROR(cudnnGetRNNWorkspaceSize(CudnnHandle(), rnn_desc, gsl::narrow_cast(seq_length), x_desc.data(), &workspace_bytes)); auto workspace_cuda = GetScratchBuffer(workspace_bytes); - int32_t zero_seq_count = 0; + int64_t zero_seq_count = 0; std::vector zero_seq_index_cache(batch_size, 0); int64_t zero_seq_index_cache_size = 0; @@ -291,8 +291,8 @@ Status CudnnRnnBase::ComputeInternal(OpKernelContext* ctx) const { if (zero_seq_count && num_directions_ > 1) { zero_seq_index_cache_size = zero_seq_count * num_directions_; zero_seq_index_cache.resize(zero_seq_index_cache_size); - for (int i = 0; i < zero_seq_count; ++i) { - zero_seq_index_cache[zero_seq_count + i] = static_cast(batch_size + zero_seq_index_cache[i]); + for (int64_t i = 0; i < zero_seq_count; ++i) { + zero_seq_index_cache[static_cast(zero_seq_count) + i] = static_cast(batch_size + zero_seq_index_cache[i]); } } diff --git a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h index 7eb4763287..8e62430980 100644 --- a/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h +++ b/onnxruntime/core/providers/cuda/rnn/cudnn_rnn_base.h @@ -24,7 +24,7 @@ enum RNN_Input_Index { }; // Onnx RNN/GRU/LSTM only support 1 layer -const int RNN_NUM_LAYERS = 1; +constexpr int RNN_NUM_LAYERS = 1; class CudnnRNN { public: diff --git a/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h b/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h index ce7e3f93d2..936a98fab6 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h +++ b/onnxruntime/core/providers/cuda/shared_inc/cuda_call.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once +#include "core/common/common.h" #include "core/providers/cuda/cuda_pch.h" namespace onnxruntime { @@ -11,7 +12,8 @@ namespace onnxruntime { // ----------------------------------------------------------------------- template -bool CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); +std::conditional_t CudaCall( + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); #define CUDA_CALL(expr) (CudaCall((expr), #expr, "CUDA", cudaSuccess)) #define CUBLAS_CALL(expr) (CudaCall((expr), #expr, "CUBLAS", CUBLAS_STATUS_SUCCESS)) diff --git a/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h b/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h index 1ec4440f00..fbe7a94166 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h +++ b/onnxruntime/core/providers/cuda/shared_inc/fpgeneric.h @@ -112,6 +112,48 @@ inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, } } +inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, int n, int k, + const float* alpha, + const half* A, int lda, + const half* B, int ldb, + const float* beta, + half* C, int ldc, + const cudaDeviceProp& prop) { + const HalfGemmOptions* half_options = HalfGemmOptions::GetInstance(); + onnxruntime::cuda::CublasMathModeSetter math_mode_setter(prop, handle, half_options->GetMathMode()); + if (half_options->IsCompute16F()) { + // The alpha and beta shall have same precision as compute type. + uint16_t h_a = onnxruntime::math::floatToHalf(*alpha); + uint16_t h_b = onnxruntime::math::floatToHalf(*beta); + return cublasGemmEx(handle, + transa, + transb, + m, n, k, + &h_a, + A, CUDA_R_16F, lda, + B, CUDA_R_16F, ldb, + &h_b, + C, CUDA_R_16F, ldc, + half_options->GetComputeType(), + CUBLAS_GEMM_DEFAULT); + } else { + return cublasGemmEx(handle, + transa, + transb, + m, n, k, + alpha, + A, CUDA_R_16F, lda, + B, CUDA_R_16F, ldb, + beta, + C, CUDA_R_16F, ldc, + half_options->GetComputeType(), + CUBLAS_GEMM_DEFAULT); + } +} + #if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 inline cublasStatus_t cublasGemmHelper(cublasHandle_t handle, cublasOperation_t transa, cublasOperation_t transb, int m, int n, int k, const BFloat16* alpha, const BFloat16* A, int lda, @@ -379,7 +421,7 @@ inline cublasStatus_t cublasGemmStridedBatchedHelper(cublasHandle_t handle, const cudaDeviceProp& prop) { const HalfGemmOptions* half_options = HalfGemmOptions::GetInstance(); onnxruntime::cuda::CublasMathModeSetter math_mode_setter(prop, handle, half_options->GetMathMode()); - if (half_options->IsCompute16F()) { + if (half_options->IsCompute16F()) { // The alpha and beta shall have same precision as compute type. uint16_t h_a = onnxruntime::math::floatToHalf(*alpha); uint16_t h_b = onnxruntime::math::floatToHalf(*beta); diff --git a/onnxruntime/core/providers/cuda/tensor/expand_impl.cu b/onnxruntime/core/providers/cuda/tensor/expand_impl.cu index fe7716696a..cadc5dccc6 100644 --- a/onnxruntime/core/providers/cuda/tensor/expand_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/expand_impl.cu @@ -59,7 +59,7 @@ __global__ void ExpandKernel( #pragma unroll for (int i = 0; i < NumElementsPerThread; i++) { if (id < N) { - // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md + // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/main/docs/Broadcasting.md CUDA_LONG index = 0; CUDA_LONG offset = id; #pragma unroll diff --git a/onnxruntime/core/providers/cuda/tensor/resize_impl.cu b/onnxruntime/core/providers/cuda/tensor/resize_impl.cu index 10232e1f15..5f6e52ff5a 100644 --- a/onnxruntime/core/providers/cuda/tensor/resize_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/resize_impl.cu @@ -694,7 +694,7 @@ void ResizeImpl( bool isSame = std::all_of(scales_vals.Data(), scales_vals.Data() + rank, [](float v) { return v == 1.0f; }) && (coordinate_transform_mode != ResizeCoordinateTransformationMode::TF_CROP_AND_RESIZE); if (isSame) { - CUDA_CALL(cudaMemcpyAsync(output_data, input_data, N * sizeof(T), cudaMemcpyDeviceToDevice, stream)); + CUDA_CALL_THROW(cudaMemcpyAsync(output_data, input_data, N * sizeof(T), cudaMemcpyDeviceToDevice, stream)); return; } diff --git a/onnxruntime/core/providers/cuda/tensor/scatter_nd.cc b/onnxruntime/core/providers/cuda/tensor/scatter_nd.cc index 3478231208..2b45b47e81 100644 --- a/onnxruntime/core/providers/cuda/tensor/scatter_nd.cc +++ b/onnxruntime/core/providers/cuda/tensor/scatter_nd.cc @@ -48,7 +48,8 @@ Status ScatterND::ComputeInternal(OpKernelContext* context) const { if (input_data != output_data) { // TODO: Run benchmarks to determine if a dedicated kernel doing data copy will be faster than invoking cudaMemcpy ? - CUDA_CALL(cudaMemcpyAsync(output_data, input_data, input_tensor->SizeInBytes(), cudaMemcpyDeviceToDevice, Stream())); + CUDA_RETURN_IF_ERROR( + cudaMemcpyAsync(output_data, input_data, input_tensor->SizeInBytes(), cudaMemcpyDeviceToDevice, Stream())); } // Bail out early diff --git a/onnxruntime/core/providers/cuda/tensor/tile.cc b/onnxruntime/core/providers/cuda/tensor/tile.cc index 6900a746ed..9fbb4f9376 100644 --- a/onnxruntime/core/providers/cuda/tensor/tile.cc +++ b/onnxruntime/core/providers/cuda/tensor/tile.cc @@ -95,8 +95,7 @@ Status Tile::ComputeInternal(OpKernelContext* ctx) const { // Repeat tensor has all 1s in it if (output_shape == input_shape) { - CUDA_CALL(cudaMemcpyAsync(output_tensor.MutableDataRaw(), input_tensor.DataRaw(), input_tensor.SizeInBytes(), cudaMemcpyDeviceToDevice, Stream())); - return Status::OK(); + return CUDA_CALL(cudaMemcpyAsync(output_tensor.MutableDataRaw(), input_tensor.DataRaw(), input_tensor.SizeInBytes(), cudaMemcpyDeviceToDevice, Stream())); } bool is_batched_memcpy = false; diff --git a/onnxruntime/core/providers/cuda/tensor/where.cc b/onnxruntime/core/providers/cuda/tensor/where.cc index 76baca5c4d..0b0c1f4a1d 100644 --- a/onnxruntime/core/providers/cuda/tensor/where.cc +++ b/onnxruntime/core/providers/cuda/tensor/where.cc @@ -115,8 +115,8 @@ struct TernaryElementwisePreparation { auto offset = out_rank - rank; for (auto i = offset; i < out_rank; ++i) { // the stride for broadcast dimension is kept as 0 - if (shape.GetDims()[i - offset] != 1) { - padded_strides[i] = pitches[i - offset]; + if (shape.GetDims()[gsl::narrow_cast(i) - offset] != 1) { + padded_strides[i] = pitches[gsl::narrow_cast(i) - offset]; } } } diff --git a/onnxruntime/core/providers/cuda/tensor/where_impl.cu b/onnxruntime/core/providers/cuda/tensor/where_impl.cu index 440a805d9a..0fbd2062ca 100644 --- a/onnxruntime/core/providers/cuda/tensor/where_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/where_impl.cu @@ -35,7 +35,7 @@ __global__ void _TenaryElementWise( #pragma unroll for (int i = 0; i < NumElementsPerThread; i++) { if (id < N) { - // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md + // compute indexes with broadcasting rules: https://github.com/onnx/onnx/blob/main/docs/Broadcasting.md CUDA_LONG cond_index = (CondIndexType == BroadcastIndexType::NoBroadcast ? id : 0); CUDA_LONG x_index = (XIndexType == BroadcastIndexType::NoBroadcast ? id : 0); CUDA_LONG y_index = (YIndexType == BroadcastIndexType::NoBroadcast ? id : 0); diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp index 255659521d..5fa9c89e6a 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/AbiCustomRegistry.cpp @@ -544,8 +544,7 @@ HRESULT STDMETHODCALLTYPE AbiCustomRegistry::RegisterOperatorKernel( // Currently unsupported for external operators if (canAliasFirstInput || supportsGraph || - requiredInputCountForGraph || - requiredConstantCpuInputs) + requiredInputCountForGraph) { ORT_THROW_HR(E_INVALIDARG); } diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp index e977900427..7c2507bb76 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/MLOperatorAuthorImpl.cpp @@ -1408,11 +1408,14 @@ namespace Windows::AI::MachineLearning::Adapter ComPtr tensor; ORT_THROW_IF_FAILED(GetInputTensor(i, tensor.GetAddressOf())); - ComPtr resource; - tensor->GetDataInterface(resource.GetAddressOf()); - if (resource) + if (tensor) { - resourcesToTransition.push_back(resource.Get()); + ComPtr resource; + tensor->GetDataInterface(resource.GetAddressOf()); + if (resource) + { + resourcesToTransition.push_back(resource.Get()); + } } } @@ -1525,21 +1528,27 @@ namespace Windows::AI::MachineLearning::Adapter ML_CHECK_BOOL(inputIndex < m_inputTensors.size()); + auto opKernelContextWrapper = const_cast(this); if (m_inputTensors[inputIndex]->GetInterface() == nullptr) { auto inputTensor = m_impl->Input(inputIndex); + if (inputTensor != nullptr) + { + ComPtr tensorWrapper = wil::MakeOrThrow( + const_cast(inputTensor), + IsAllocationInterface(inputTensor->Location()), + m_winmlProvider.Get(), + m_internalOperator); - ComPtr tensorWrapper = wil::MakeOrThrow( - const_cast(inputTensor), - IsAllocationInterface(inputTensor->Location()), - m_winmlProvider.Get(), - m_internalOperator); - - const_cast(this)->m_inputTensors[inputIndex] = tensorWrapper; + opKernelContextWrapper->m_inputTensors[inputIndex] = tensorWrapper; + } } - const_cast(this)->m_inputTensors[inputIndex].CopyTo(tensor); + if (opKernelContextWrapper->m_inputTensors[inputIndex] != nullptr) + { + opKernelContextWrapper->m_inputTensors[inputIndex].CopyTo(tensor); + } return S_OK; } ORT_CATCH_RETURN diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h new file mode 100644 index 0000000000..ca8ed721fd --- /dev/null +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlDFT.h @@ -0,0 +1,738 @@ +#pragma once + +#include "../MLOperatorAuthorImpl.h" +#include "../../../OperatorAuthorHelper/OperatorHelper.h" + +#include "../External/D3DX12/d3dx12.h" + +// The shader header is produced using "fxc.exe dft_shader.hlsl -E DFT -T cs_5_0 -Zi /Fh" +#include "GeneratedShaders/stockham.h" + +#include +#include + +#include + +using namespace Microsoft::WRL; + +namespace DFTHelpers { + // Divides and rounds up + inline uint32_t CeilDivide(uint32_t dividend, uint32_t divisor) + { + UINT64 temp = static_cast(dividend) + divisor - 1; + return static_cast(temp / divisor); + } + + // Gets the next number of elements to dispatch to the GPU within a loop handling a large + // total number of tensor elements and threads. + void GetNextDispatchSize( + uint32_t elementCount, + uint32_t elementsPerThread, + uint32_t numThreads, + _Out_ uint32_t& dispatch, + _Out_ uint32_t& pendingElementCount + ) + { + // Max threads per workgroup is 2^10 (1024). Max dispatch per dimension is 2^16. Taken together, we can dispatch a maximum of + // 2^26 (268,435,456) threads along a single dimension. This should suffice for a majority of the workload. Therefore, even + // though it is possible to dispatch up to (2^16)^3 workgroups simultaneously, we stick to the simpler 1D dispatch alternative. + assert(numThreads <= D3D12_CS_THREAD_GROUP_MAX_THREADS_PER_GROUP); + + const uint32_t maxThreadsPerDispatch = numThreads * D3D12_CS_DISPATCH_MAX_THREAD_GROUPS_PER_DIMENSION; + + const uint32_t requiredThreadCount = CeilDivide(elementCount, elementsPerThread); + + // Compute max dispatchable elements + const uint32_t availableThreadCount = std::min(requiredThreadCount, maxThreadsPerDispatch); + + // Compute required thread group count + uint32_t workGroupCount1D = CeilDivide(availableThreadCount, numThreads); + + // Compute min dispatch size + dispatch = workGroupCount1D; + + // With the dispatch size computed, compute the dispatched element count + const uint32_t dispatchedElementCount = workGroupCount1D * numThreads * elementsPerThread; + + // Update the pending element count + pendingElementCount = (dispatchedElementCount < elementCount) ? elementCount - dispatchedElementCount : 0; + } + +} + +class GpuDFTOperator : public WRL::Base +{ +private: + ComPtr m_device; + ComPtr m_rootSignature; + ComPtr m_pipelineState; + + int64_t m_axis; + bool m_isOnesided; + bool m_isInverse; + + struct StockhamParameters + { + // Allocate temporary buffers if needed + struct ResourceDesc + { + ComPtr Resource; + std::array Sizes; + std::array Strides; + }; + + struct LoopRangeCalculator + { + unsigned Left; + unsigned Right; + unsigned End; + unsigned CalculateIndex(unsigned index) const + { + if (index > 0 && index < End) + { + unsigned range = Right - Left + 1; + index = Left + (index - 1) % range; + } + else if (index == End) + { + index = Right + 1; + } + return index; + } + }; + + std::vector ResourceLoopList = {}; + LoopRangeCalculator LoopRange = {}; + uint32_t OutputIndex = 0; + uint32_t NumberOfPasses = 0; + }; + + enum class DFTType + { + Stockham = 0, + BluesteinZChirp, + }; + + struct DFTParameters + { + DFTType Type = DFTType::Stockham; + StockhamParameters StockhamParams = {}; + uint32_t DFTLength = 0; + }; + + struct DFTShaderConstants + { + uint32_t StartIndex; + uint32_t ElementCount; + uint32_t DFTIteration; + uint32_t IsInverse; + uint32_t InputSizes[4]; + uint32_t InputStrides[4]; + uint32_t OutputSizes[4]; + uint32_t OutputStrides[4]; + float Scale; + uint32_t DFTLength; + }; + +public: + GpuDFTOperator(IMLOperatorKernelCreationContext* context) + { + ComPtr executionObject; + context->GetExecutionInterface(executionObject.GetAddressOf()); + + ComPtr commandList; + executionObject.As(&commandList); + + ORT_THROW_IF_FAILED(commandList->GetDevice(IID_ID3D12Device, &m_device)); + + ORT_THROW_IF_FAILED(context->GetAttribute("axis", MLOperatorAttributeType::Int, 1, sizeof(int64_t), reinterpret_cast(&m_axis))); + + int64_t isInverseInt; + ORT_THROW_IF_FAILED(context->GetAttribute("inverse", MLOperatorAttributeType::Int, 1, sizeof(int64_t), reinterpret_cast(&isInverseInt))); + m_isInverse = static_cast(isInverseInt); + + int64_t isOnesidedInt; + ORT_THROW_IF_FAILED(context->GetAttribute("onesided", MLOperatorAttributeType::Int, 1, sizeof(int64_t), reinterpret_cast(&isOnesidedInt))); + m_isOnesided = static_cast(isOnesidedInt); + + PrepareGpuResources(); + } + + void PrepareGpuResources() + { + // Compute root signature. + const int uavCount = 2; + std::vector rootParameters; + rootParameters.resize(uavCount + 1); + + for (UINT i = 0; i < uavCount; i++) + { + rootParameters[i].InitAsUnorderedAccessView(i); + } + + int constantCount = 22; + rootParameters[uavCount].InitAsConstants(constantCount, 0); + + CD3DX12_VERSIONED_ROOT_SIGNATURE_DESC desc; + desc.Init_1_1(static_cast(rootParameters.size()), rootParameters.data()); + + ComPtr rootSignatureBlob; + ComPtr rootSignatureErrorBlob; + ORT_THROW_IF_FAILED(D3D12SerializeVersionedRootSignature( + &desc, + rootSignatureBlob.GetAddressOf(), + rootSignatureErrorBlob.GetAddressOf() + )); + + ORT_THROW_IF_FAILED(m_device->CreateRootSignature( + 0, + rootSignatureBlob->GetBufferPointer(), + rootSignatureBlob->GetBufferSize(), + IID_ID3D12RootSignature, + &m_rootSignature + )); + + // Describe and create the compute pipeline state object (PSO). + D3D12_COMPUTE_PIPELINE_STATE_DESC computePsoDesc = {}; + computePsoDesc.pRootSignature = m_rootSignature.Get(); + computePsoDesc.CS = CD3DX12_SHADER_BYTECODE(g_DFT, sizeof(g_DFT)); + + ORT_THROW_IF_FAILED(m_device->CreateComputePipelineState(&computePsoDesc, IID_ID3D12PipelineState, &m_pipelineState)); + } + + // Computes the outputs of the kernel. This may be called multiple times + // simultaneously within the same instance of the class. Implementations + // of this method must be thread-safe. + STDMETHOD(Compute)(IMLOperatorKernelContext* context) + { + try + { + // Get the input tensor + ComPtr inputTensor; + ORT_THROW_IF_FAILED(context->GetInputTensor(0, inputTensor.GetAddressOf())); + + // Get the output tensor + ComPtr outputTensor; + context->GetOutputTensor(0, outputTensor.GetAddressOf()); + + if (outputTensor->IsCpuData() || inputTensor->IsCpuData()) + { + return E_UNEXPECTED; + } + + if (outputTensor->GetTensorDataType() != MLOperatorTensorDataType::Float || + inputTensor->GetTensorDataType() != MLOperatorTensorDataType::Float) + { + return E_UNEXPECTED; + } + + ComPtr executionObject; + ComPtr commandList; + context->GetExecutionInterface(executionObject.GetAddressOf()); + executionObject.As(&commandList); + + auto dftParams = PrepareDFT(context); + switch (dftParams.Type) + { + case DFTType::Stockham: + { + StockhamFFT(dftParams, commandList.Get()); + break; + } + case DFTType::BluesteinZChirp: + __fallthrough; + default: + return E_NOTIMPL; + } + } + catch (...) + { + return E_FAIL; + } + + return S_OK; + } + + DFTParameters PrepareDFT(IMLOperatorKernelContext* context) + { + DFTParameters params = {}; + + // Get the input and output shape sizes + ComPtr inputTensor; + ORT_THROW_IF_FAILED(context->GetInputTensor(0, &inputTensor)); + auto inputDims = GetTensorDimensions(inputTensor.Get()); + auto inputDataSize = ComputeElementCountFromDimensions(inputDims); + ML_CHECK_VALID_ARGUMENT(static_cast(m_axis) < inputDims.size()) + + ComPtr outputTensor; + ORT_THROW_IF_FAILED(context->GetOutputTensor(0, &outputTensor)); + auto outputDims = GetTensorDimensions(outputTensor.Get()); + auto outputDataSize = ComputeElementCountFromDimensions(outputDims); + + ORT_THROW_HR_IF(E_FAIL, inputDims.size() != outputDims.size()); + + // Get optional dft_length input + ComPtr dftLengthTensor; + if (SUCCEEDED(context->GetInputTensor(1, &dftLengthTensor)) && dftLengthTensor != nullptr) + { + MLOperatorTensor tensor(dftLengthTensor.Get()); + params.DFTLength = gsl::narrow_cast(OperatorHelper::ReadScalarTensorCastToInt64(tensor)); + } + else + { + params.DFTLength = inputDims[m_axis]; + } + + auto isPowerOfTwo = [](uint32_t n) { return (n != 0) && ((n & (n - 1)) == 0); }; + if (!isPowerOfTwo(params.DFTLength)) + { + params.Type = DFTType::BluesteinZChirp; + params.StockhamParams = {}; + } + else + { + params.Type = DFTType::Stockham; + params.StockhamParams = {}; + + ComPtr inputUnknown; + ComPtr inputResource; + inputTensor->GetDataInterface(inputUnknown.GetAddressOf()); + inputUnknown.As(&inputResource); + + ComPtr outputUnknown; + ComPtr outputResource; + outputTensor->GetDataInterface(outputUnknown.GetAddressOf()); + outputUnknown.As(&outputResource); + + // { before_dft_axis, axis, after_dft_axis, real_or_complex } + std::array reshapedInputSize = { 1, 1, 1, inputDims.back() }; + std::array reshapedOutputSize = { 1, 1, 1, outputDims.back() }; + + size_t reshapedIndex = 0; + for (int i = 0; i < inputDims.size() - 1; i++) + { + if (i == m_axis || i == (m_axis + 1)) + { + reshapedIndex++; + } + reshapedInputSize[reshapedIndex] *= inputDims[i]; + reshapedOutputSize[reshapedIndex] *= outputDims[i]; + } + + auto temporarySize = reshapedInputSize; + temporarySize.back() = reshapedOutputSize.back(); + auto temporaryBufferByteSize = sizeof(float) * ComputeElementCountFromDimensions(temporarySize); + + // Calculate elements and strides + std::array reshapedInputStrides = { 1, 1, 1, 1 }; + std::array reshapedOutputStrides = { 1, 1, 1, 1 }; + std::array temporaryStrides = { 1, 1, 1, 1 }; + for (int i = static_cast(reshapedInputSize.size()) - 2; i >= 0; i--) + { + reshapedInputStrides[i] = reshapedInputSize[i + 1] * reshapedInputStrides[i + 1]; + reshapedOutputStrides[i] = reshapedOutputSize[i + 1] * reshapedOutputStrides[i + 1]; + temporaryStrides[i] = temporarySize[i + 1] * temporaryStrides[i + 1]; + } + + // Calculate passes + params.StockhamParams.NumberOfPasses = static_cast(log2(params.DFTLength)); + bool hasOnePass = params.StockhamParams.NumberOfPasses == 1; + bool hasOddPasses = params.StockhamParams.NumberOfPasses % 2; + bool hasEvenPasses = !hasOddPasses; + + // write directly input buffer to output buffer, dont create temps + bool writeToOutput = hasOnePass; + // First and final are input/output buffers, but all else ocillate between 2 temp buffers + bool oscillateBetweenTwoTemporaries = !hasOnePass && m_isOnesided; + // First is input buffer, all else ocillate between temp and output, causing the final pass to write to the output buffer + bool oscillateFirstOutputThenTemporary = hasOddPasses && !m_isOnesided; + // First is input buffer, all else ocillate between output and temp, causing the final pass to write to the output buffer + bool oscillateFirstTemporaryThenOutput = hasEvenPasses && !m_isOnesided; + + // Create the resource loop list + // Add the input resource to the loop list + params.StockhamParams.ResourceLoopList.push_back({}); + params.StockhamParams.ResourceLoopList.back().Resource = inputResource; + params.StockhamParams.ResourceLoopList.back().Sizes = reshapedInputSize; + params.StockhamParams.ResourceLoopList.back().Strides = reshapedInputStrides; + + // If 1 temporary should be placed first, or multiple temporaries, then + // Add a temp in the list + if (oscillateFirstTemporaryThenOutput || oscillateBetweenTwoTemporaries) + { + params.StockhamParams.ResourceLoopList.push_back({}); + params.StockhamParams.ResourceLoopList.back().Sizes = temporarySize; + params.StockhamParams.ResourceLoopList.back().Strides = temporaryStrides; + + auto& resource = params.StockhamParams.ResourceLoopList.back().Resource; + ORT_THROW_IF_FAILED(context->AllocateTemporaryData(temporaryBufferByteSize, &resource)); + } + + // If 2 temps, add another + if (oscillateBetweenTwoTemporaries) + { + params.StockhamParams.ResourceLoopList.push_back({}); + params.StockhamParams.ResourceLoopList.back().Sizes = temporarySize; + params.StockhamParams.ResourceLoopList.back().Strides = temporaryStrides; + + auto& resource = params.StockhamParams.ResourceLoopList.back().Resource; + ORT_THROW_IF_FAILED(context->AllocateTemporaryData(temporaryBufferByteSize, &resource)); + } + + // Add output resource + params.StockhamParams.ResourceLoopList.push_back({}); + params.StockhamParams.ResourceLoopList.back().Resource = outputResource; + params.StockhamParams.ResourceLoopList.back().Sizes = reshapedOutputSize; + params.StockhamParams.ResourceLoopList.back().Strides = reshapedOutputStrides; + params.StockhamParams.OutputIndex = static_cast(params.StockhamParams.ResourceLoopList.size() - 1); + + // Add the temporary after output incase of odd number of passes + if (oscillateFirstOutputThenTemporary) + { + params.StockhamParams.ResourceLoopList.push_back({}); + params.StockhamParams.ResourceLoopList.back().Sizes = temporarySize; + params.StockhamParams.ResourceLoopList.back().Strides = temporaryStrides; + + auto& resource = params.StockhamParams.ResourceLoopList.back().Resource; + ORT_THROW_IF_FAILED(context->AllocateTemporaryData(temporaryBufferByteSize, &resource)); + } + + // Define the loop range + if (writeToOutput) { params.StockhamParams.LoopRange = { 0, 1, params.StockhamParams.NumberOfPasses }; } + if (oscillateBetweenTwoTemporaries) { params.StockhamParams.LoopRange = { 1, 2, params.StockhamParams.NumberOfPasses }; } + if (oscillateFirstOutputThenTemporary) { params.StockhamParams.LoopRange = { 1, 2, params.StockhamParams.NumberOfPasses + 1 }; } + if (oscillateFirstTemporaryThenOutput) { params.StockhamParams.LoopRange = { 1, 2, params.StockhamParams.NumberOfPasses + 1 }; } + } + + return params; + } + + void StockhamFFT(const DFTParameters& dftParams, ID3D12GraphicsCommandList* commandList) + { + const auto& stockhamParams = dftParams.StockhamParams; + + // Create resource loop list + const auto& loopList = stockhamParams.ResourceLoopList; + + // Get input and output resources + auto inputResource = loopList[0].Resource.Get(); + auto outputResource = loopList[stockhamParams.OutputIndex].Resource.Get(); + + // Transition resources from common to UAV state + D3D12_RESOURCE_BARRIER barriers[2]; + + barriers[0] = CD3DX12_RESOURCE_BARRIER::Transition( + inputResource, + D3D12_RESOURCE_STATE_COMMON, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS + ); + + barriers[1] = CD3DX12_RESOURCE_BARRIER::Transition( + outputResource, + D3D12_RESOURCE_STATE_COMMON, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS + ); + + commandList->ResourceBarrier(2, barriers); + + // Set the root signature and pipeline state + commandList->SetComputeRootSignature(m_rootSignature.Get()); + commandList->SetPipelineState(m_pipelineState.Get()); + + // Each iteration of the below loop represents 1 level in the Stockham DFT + // Dispatch in a loop + DFTShaderConstants constants = {}; + constants.DFTLength = dftParams.DFTLength; + constants.DFTIteration = 0; + constants.IsInverse = m_isInverse; + + for (unsigned index = 0; index < stockhamParams.NumberOfPasses; index++) + { + auto inIdx = stockhamParams.LoopRange.CalculateIndex(index); + auto outIdx = stockhamParams.LoopRange.CalculateIndex(index + 1); + + auto in = loopList[inIdx].Resource.Get(); + std::copy(loopList[inIdx].Sizes.begin(), loopList[inIdx].Sizes.end(), constants.InputSizes); + std::copy(loopList[inIdx].Strides.begin(), loopList[inIdx].Strides.end(), constants.InputStrides); + + auto out = loopList[outIdx].Resource.Get(); + std::copy(loopList[outIdx].Sizes.begin(), loopList[outIdx].Sizes.end(), constants.OutputSizes); + std::copy(loopList[outIdx].Strides.begin(), loopList[outIdx].Strides.end(), constants.OutputStrides); + + auto isLastPass = (index == stockhamParams.NumberOfPasses - 1); + auto isLastInversePass = isLastPass && m_isInverse; + auto dftLength = 1 << stockhamParams.NumberOfPasses; + constants.Scale = isLastInversePass ? (1.f / dftLength) : 1.f; + + auto totalElementCount = + std::accumulate(constants.OutputSizes, + constants.OutputSizes + std::size(constants.OutputSizes), + 1, + std::multiplies()); + constants.ElementCount = totalElementCount / constants.OutputSizes[3]; + constants.DFTIteration = index + 1; + Dispatch(in, out, constants, commandList); + } + + // Transition resources to common state + barriers[0] = CD3DX12_RESOURCE_BARRIER::Transition( + inputResource, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_STATE_COMMON + ); + + barriers[1] = CD3DX12_RESOURCE_BARRIER::Transition( + outputResource, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_STATE_COMMON + ); + + commandList->ResourceBarrier(2, barriers); + } + + std::vector GetTensorDimensions(IMLOperatorTensor* tensor) + { + auto inputDimsSize = tensor->GetDimensionCount(); + auto dims = std::vector(inputDimsSize); + ORT_THROW_IF_FAILED(tensor->GetShape(static_cast(dims.size()), dims.data())); + return dims; + } + + void Dispatch( + ID3D12Resource* inputResource, + ID3D12Resource* outputResource, + DFTShaderConstants& constants, + ID3D12GraphicsCommandList* commandList) + { + D3D12_RESOURCE_BARRIER uav_barriers[2]; + uav_barriers[0] = CD3DX12_RESOURCE_BARRIER::UAV(inputResource); + uav_barriers[1] = CD3DX12_RESOURCE_BARRIER::UAV(outputResource); + commandList->ResourceBarrier(2, uav_barriers); + // Set resource views + commandList->SetComputeRootUnorderedAccessView( + 0, // root parameter index + inputResource->GetGPUVirtualAddress() + ); + + commandList->SetComputeRootUnorderedAccessView( + 1, // root parameter index + outputResource->GetGPUVirtualAddress() + ); + auto pendingElementCount = constants.ElementCount; + + // Dispatch up to the maximum number of threads per iteration until + // all elements are completed + while (pendingElementCount > 0) + { + constants.StartIndex = constants.ElementCount - pendingElementCount; + + uint32_t dispatchSizeX; + + DFTHelpers::GetNextDispatchSize( + pendingElementCount, + 1, + 64, + dispatchSizeX, + pendingElementCount + ); + + // Set root constants + commandList->SetComputeRoot32BitConstants( + 2, // root parameter index + 22, // Constant count + &constants, + 0 // offset + ); + + commandList->Dispatch(dispatchSizeX, 1, 1); + } + + commandList->ResourceBarrier(2, uav_barriers); + } +}; + +struct DFTShapeInferrer : public WRL::Base +{ + STDMETHOD(InferOutputShapes)(IMLOperatorShapeInferenceContext* context) noexcept + { + try + { + int64_t axis; + ORT_THROW_IF_FAILED(context->GetAttribute("axis", MLOperatorAttributeType::Int, 1, sizeof(int64_t), reinterpret_cast(&axis))); + int64_t isInverseInt; + ORT_THROW_IF_FAILED(context->GetAttribute("inverse", MLOperatorAttributeType::Int, 1, sizeof(int64_t), reinterpret_cast(&isInverseInt))); + int64_t isOnesidedInt; + ORT_THROW_IF_FAILED(context->GetAttribute("onesided", MLOperatorAttributeType::Int, 1, sizeof(int64_t), reinterpret_cast(&isOnesidedInt))); + bool isOnesided = static_cast(isOnesidedInt); + bool isInverse = static_cast(isInverseInt); + + if (isInverse && isOnesided) + { + throw new std::exception("onesided and inverse attributes cannot be enabled at the same time"); + } + + uint32_t rank; + ORT_THROW_IF_FAILED(context->GetInputTensorDimensionCount(0, &rank)); + if (rank == 0) + { + // If no shape is available for the input, skip shape inference... + throw; + } + + auto axisIdx = OperatorHelper::HandleNegativeAxis(static_cast(axis), rank); + + // In general the output shape will match the input shape exactly + // So initialize the output shape with the input shape + std::vector inputDims(rank); + ORT_THROW_IF_FAILED(context->GetInputTensorShape(0, rank, inputDims.data())); + auto outputDims = inputDims; + // The last dimension of the output shape is always 2. + // It corresponds to the real and imaginary parts of the DFT output. + outputDims.back() = 2; + + if (context->IsInputValid(1)) + { + // If dft_length is specified, then we should honor the shape. + // If onesided this will be adjusted later on. + ComPtr contextPrivate; + ORT_THROW_IF_FAILED(context->QueryInterface(IID_PPV_ARGS(&contextPrivate))); + ComPtr dftLengthTensor; + ORT_THROW_IF_FAILED(contextPrivate->GetConstantInputTensor(1, &dftLengthTensor)); + MLOperatorTensor tensor(dftLengthTensor.Get()); + auto dftLength = gsl::narrow_cast(OperatorHelper::ReadScalarTensorCastToInt64(tensor)); + outputDims[axisIdx] = dftLength; + } + + // When DFT is onesided, the output shape is half the size of the input shape + // along the specified axis. + if (isOnesided) + { + auto axisDimension = outputDims.at(axisIdx); + // We need to update the output shape dimension along the specified axis, + // but sometimes the dimension will be a free dimension or be otherwise unset. + // Only perform inference when a input dimension value exists. + auto originalSignalSize = axisDimension; + auto halfSignalSize = (originalSignalSize >> 1) + 1; + outputDims.at(axisIdx) = halfSignalSize; + } + + ORT_THROW_IF_FAILED(context->SetOutputTensorShape(0, rank, outputDims.data())); + } + catch (...) + { + return E_FAIL; + } + + return S_OK; + } +}; + +class GpuDFTOperatorFactory : public WRL::Base +{ +public: + STDMETHOD(CreateKernel)( + IMLOperatorKernelCreationContext* context, + IMLOperatorKernel** kernel) + { + try + { + auto dftOperator = wil::MakeOrThrow(context); + dftOperator.CopyTo(kernel); + return S_OK; + } + catch (...) + { + return E_FAIL; + } + } + + static void RegisterDFTKernel(IMLOperatorRegistry* registry) + { + MLOperatorKernelDescription kernelDescription = {}; + kernelDescription.domain = ""; + kernelDescription.name = "DFT"; + kernelDescription.minimumOperatorSetVersion = 17; + kernelDescription.executionType = MLOperatorExecutionType::D3D12; + + // T1: tensor(float16), tensor(float), tensor(double), tensor(bfloat16) + MLOperatorEdgeTypeConstrant t1Constraint; + t1Constraint.typeLabel = "T1"; + std::vector t1AllowedEdges + { + //MLOperatorEdgeDescription { MLOperatorEdgeType::Tensor, (uint64_t)MLOperatorTensorDataType::Float16 }, + MLOperatorEdgeDescription { MLOperatorEdgeType::Tensor, (uint64_t)MLOperatorTensorDataType::Float }, + //MLOperatorEdgeDescription { MLOperatorEdgeType::Tensor, (uint64_t)MLOperatorTensorDataType::Double }, + }; + t1Constraint.allowedTypes = t1AllowedEdges.data(); + t1Constraint.allowedTypeCount = static_cast(t1AllowedEdges.size()); + + // T2 : tensor(int32), tensor(int64) + MLOperatorEdgeTypeConstrant t2Constraint; + t2Constraint.typeLabel = "T2"; + std::vector t2AllowedEdges + { + // MLOperatorEdgeDescription { MLOperatorEdgeType::Tensor, (uint64_t)MLOperatorTensorDataType::Int32 }, + MLOperatorEdgeDescription { MLOperatorEdgeType::Tensor, (uint64_t)MLOperatorTensorDataType::Int64 }, + }; + t2Constraint.allowedTypes = t2AllowedEdges.data(); + t2Constraint.allowedTypeCount = static_cast(t2AllowedEdges.size()); + + std::vector typeConstraints{ t1Constraint, t2Constraint }; + kernelDescription.typeConstraints = typeConstraints.data(); + kernelDescription.typeConstraintCount = static_cast(typeConstraints.size()); + + MLOperatorAttributeNameValue axisAttributeValue; + axisAttributeValue.name = "axis"; + axisAttributeValue.type = MLOperatorAttributeType::Int; + axisAttributeValue.valueCount = 1; + static const int64_t axis[] = { 1 }; + axisAttributeValue.ints = axis; + + MLOperatorAttributeNameValue inverseAttributeValue; + inverseAttributeValue.name = "inverse"; + inverseAttributeValue.type = MLOperatorAttributeType::Int; + inverseAttributeValue.valueCount = 1; + static const int64_t inverse[] = { 0 }; + inverseAttributeValue.ints = inverse; + + MLOperatorAttributeNameValue onesidedAttributeValue; + onesidedAttributeValue.name = "onesided"; + onesidedAttributeValue.type = MLOperatorAttributeType::Int; + onesidedAttributeValue.valueCount = 1; + static const int64_t onesided[] = { 0 }; + onesidedAttributeValue.ints = onesided; + + std::vector attributeDefaultValues{ + axisAttributeValue, + inverseAttributeValue, + onesidedAttributeValue + }; + + kernelDescription.defaultAttributes = attributeDefaultValues.data(); + kernelDescription.defaultAttributeCount = static_cast(attributeDefaultValues.size()); + kernelDescription.options = MLOperatorKernelOptions::None; + kernelDescription.executionOptions = 0; + + auto shareInferrer = wil::MakeOrThrow(); + auto factory = wil::MakeOrThrow(); + + std::array requiredConstantCpuInputs = { 1 }; + + ComPtr registryPrivate; + ORT_THROW_IF_FAILED(registry->QueryInterface(IID_PPV_ARGS(®istryPrivate))); + + ORT_THROW_IF_FAILED(registryPrivate->RegisterOperatorKernel( + &kernelDescription, + factory.Get(), + shareInferrer.Get(), + nullptr, + false, // isInternalOperator + false, // alias + false, // supportsGraph + nullptr, + requiredConstantCpuInputs.data(), + static_cast(requiredConstantCpuInputs.size()) + )); + + } +}; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorInstanceNormalization.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorInstanceNormalization.cpp index d41f90ac41..a90d5b9e38 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorInstanceNormalization.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorInstanceNormalization.cpp @@ -15,18 +15,21 @@ class DmlOperatorInstanceNormalization : public DmlOperator IN_BIAS }; - void Shift1DInputsTensorDesc(const MLOperatorKernelCreationContext& kernelCreationContext, int index, int count, uint32_t destinationAxis) + void Shift1DInputsTensorDesc( + const MLOperatorKernelCreationContext& kernelCreationContext, + int index, + int count, + uint32_t minimumDimensionCount + ) { for (int i = index; i != index + count; ++i) { // Shift a single dimension size to the C channel. - // e.g. [7] or [1,1,1,7] becomes [1,7,1,1] + // e.g. Given a 4D input (X), then a 1D scale of [7] becomes [1,7,1,1]. TensorDesc& tensorDesc = m_inputTensorDescs[i]; gsl::span sizes = tensorDesc.GetSizes(); gsl::span lastDimension = sizes.last(1); - ML_CHECK_VALID_ARGUMENT(tensorDesc.GetDimensionCount() == OperatorHelper::NchwDimensionCount); - ML_CHECK_VALID_ARGUMENT(sizes.size() >=4 && sizes[N] == 1 && sizes[C] == 1 && sizes[H] == 1); - m_inputTensorDescs[i] = CreateTensorDescFromInput(kernelCreationContext, i, TensorAxis::DoNotCoerce, TensorAxis::C, TensorAxis::LeftAligned, lastDimension); + m_inputTensorDescs[i] = CreateTensorDescFromInput(kernelCreationContext, i, TensorAxis::DoNotCoerce, TensorAxis::C, TensorAxis::LeftAligned, lastDimension, minimumDimensionCount); } } @@ -35,29 +38,46 @@ public: : DmlOperator(kernelCreationContext) { std::vector> kernelInputIndices = {0, 1, 2}; - DmlOperator::Initialize(kernelCreationContext, kernelInputIndices); + DmlOperator::Initialize( + kernelCreationContext, + kernelInputIndices, + std::nullopt, + std::nullopt, + std::nullopt, + /*minimumDimensionCount*/ 1 + ); + + // "Instance" normalization is really spatial normalization, + // where the spatial channels are reduced and normalized, while + // batch and channel remain independent. So pass a list of axes + // just beyond the leading batch and channel dimensions (starting + // at axis 2 up to the last spatial dimension). + const uint32_t inputDimensionCount = m_inputTensorDescs.front().GetDimensionCount(); + std::vector axes(inputDimensionCount - 2); + std::iota(axes.begin(), axes.end(), 2u); const float epsilon = kernelCreationContext.GetOptionalAttribute(AttrName::Epsilon, DefaultEpsilon); const std::optional fusedActivation = FusionHelpers::TryGetFusedActivationDesc(kernelCreationContext); DML_OPERATOR_DESC fusedActivationDmlDesc = fusedActivation ? fusedActivation->GetDmlDesc() : DML_OPERATOR_DESC(); // Shift IN_SCALE and IN_BIAS input tensor descs {1, C, 1, 1} out of 1D tensors. - Shift1DInputsTensorDesc(kernelCreationContext, IN_SCALE, 2, C); + Shift1DInputsTensorDesc(kernelCreationContext, IN_SCALE, 2, inputDimensionCount); std::vector inputDescs = GetDmlInputDescs(); std::vector outputDescs = GetDmlOutputDescs(); - DML_MEAN_VARIANCE_NORMALIZATION_OPERATOR_DESC operatorDesc = {}; + DML_MEAN_VARIANCE_NORMALIZATION1_OPERATOR_DESC operatorDesc = {}; operatorDesc.InputTensor = &inputDescs[0]; operatorDesc.ScaleTensor = &inputDescs[1]; operatorDesc.BiasTensor = &inputDescs[2]; operatorDesc.OutputTensor = outputDescs.data(); - operatorDesc.CrossChannel = false; + operatorDesc.Axes = axes.data(); + operatorDesc.AxisCount = static_cast(axes.size()); operatorDesc.NormalizeVariance = true; operatorDesc.Epsilon = epsilon; operatorDesc.FusedActivation = fusedActivation ? &fusedActivationDmlDesc : nullptr; - DML_OPERATOR_DESC opDesc = { DML_OPERATOR_MEAN_VARIANCE_NORMALIZATION, &operatorDesc }; + DML_OPERATOR_DESC opDesc = { DML_OPERATOR_MEAN_VARIANCE_NORMALIZATION1, &operatorDesc }; SetDmlOperatorDesc(opDesc, kernelCreationContext); } }; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorValueScale2D.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorValueScale2D.cpp index 3dda22db9b..6dc039e468 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorValueScale2D.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/DmlOperatorValueScale2D.cpp @@ -6,7 +6,7 @@ namespace Dml { -// https://github.com/onnx/onnx/blob/master/docs/Operators.md#experimental-imagescaler +// https://github.com/onnx/onnx/blob/main/docs/Operators.md#experimental-imagescaler class DmlOperatorValueScale2d : public DmlOperator { public: diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/GeneratedShaders/GenerateShaders.bat b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/GeneratedShaders/GenerateShaders.bat new file mode 100644 index 0000000000..4bfffb11dd --- /dev/null +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/GeneratedShaders/GenerateShaders.bat @@ -0,0 +1 @@ +fxc.exe ..\Shaders\stockham.hlsl -E DFT -T cs_5_0 -Zi /Od /Fh stockham.h \ No newline at end of file diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/GeneratedShaders/stockham.h b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/GeneratedShaders/stockham.h new file mode 100644 index 0000000000..c9e9b0c626 --- /dev/null +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/GeneratedShaders/stockham.h @@ -0,0 +1,5303 @@ +#if 0 +// +// Generated by Microsoft (R) HLSL Shader Compiler 10.1 +// +// +// Buffer Definitions: +// +// cbuffer Constants +// { +// +// uint StartIndex; // Offset: 0 Size: 4 +// uint ElementCount; // Offset: 4 Size: 4 +// uint DFTIteration; // Offset: 8 Size: 4 +// uint IsInverse; // Offset: 12 Size: 4 +// uint4 InputSizes; // Offset: 16 Size: 16 +// uint4 InputStrides; // Offset: 32 Size: 16 +// uint4 OutputSizes; // Offset: 48 Size: 16 +// uint4 OutputStrides; // Offset: 64 Size: 16 +// float Scale; // Offset: 80 Size: 4 +// uint DFTLength; // Offset: 84 Size: 4 +// +// } +// +// Resource bind info for src +// { +// +// float $Element; // Offset: 0 Size: 4 +// +// } +// +// Resource bind info for dst +// { +// +// float $Element; // Offset: 0 Size: 4 +// +// } +// +// +// Resource Bindings: +// +// Name Type Format Dim HLSL Bind Count +// ------------------------------ ---------- ------- ----------- -------------- ------ +// src UAV struct r/w u0 1 +// dst UAV struct r/w u1 1 +// Constants cbuffer NA NA cb0 1 +// +// +// +// Input signature: +// +// Name Index Mask Register SysValue Format Used +// -------------------- ----- ------ -------- -------- ------- ------ +// no Input +// +// Output signature: +// +// Name Index Mask Register SysValue Format Used +// -------------------- ----- ------ -------- -------- ------- ------ +// no Output +cs_5_0 +dcl_globalFlags refactoringAllowed | skipOptimization +dcl_constantbuffer CB0[6], immediateIndexed +dcl_uav_structured u0, 4 +dcl_uav_structured u1, 4 +dcl_input vThreadID.x +dcl_temps 5 +dcl_thread_group 64, 1, 1 +// +// Initial variable locations: +// vThreadID.x <- dtid.x; vThreadID.y <- dtid.y; vThreadID.z <- dtid.z +// +#line 87 "E:\work\onnxruntime3\onnxruntime\core\providers\dml\DmlExecutionProvider\src\Operators\Shaders\stockham.hlsl" +mov r0.x, l(6.283185) // r0.x <- TAU + +#line 61 +iadd r0.y, vThreadID.x, cb0[0].x // r0.y <- index + +#line 62 +ult r0.z, r0.y, cb0[0].y +if_nz r0.z + +#line 64 + mov r0.z, l(2) + udiv r0.z, null, cb0[5].y, r0.z // r0.z <- halfTotalDFTLength + +#line 65 + mov r0.w, cb0[0].z + ishl r0.w, l(1), r0.w // r0.w <- N + +#line 66 + mov r1.x, l(1) + ineg r1.x, r1.x + iadd r1.x, r1.x, cb0[0].z + ishl r1.x, l(1), r1.x // r1.x <- halfN + +#line 71 + nop + mov r0.y, r0.y + +#line 49 + imul null, r1.y, cb0[3].z, cb0[3].y + udiv null, r1.y, r0.y, r1.y // r1.y <- temp + +#line 52 + imul null, r1.z, cb0[3].z, cb0[3].y + udiv r2.x, null, r0.y, r1.z // r2.x <- idx.x + +#line 53 + udiv r2.y, null, r1.y, cb0[3].z // r2.y <- idx.y + +#line 54 + udiv null, r2.z, r1.y, cb0[3].z // r2.z <- idx.z + +#line 55 + mov r2.x, r2.x // r2.x <- .x + mov r2.y, r2.y // r2.y <- .y + mov r2.z, r2.z // r2.z <- .z + +#line 71 + mov r2.xyz, r2.xyzx // r2.x <- idx.x; r2.y <- idx.y; r2.z <- idx.z + +#line 72 + ushr r1.y, r2.y, cb0[0].z + imul null, r1.y, r1.x, r1.y + udiv null, r1.x, r2.y, r1.x + iadd r1.y, r1.x, r1.y // r1.y <- inputEvenOddIndexPair.x + +#line 73 + iadd r3.y, r0.z, r1.y // r3.y <- inputEvenOddIndexPair.y + +#line 76 + mov r1.xz, r2.xxzx // r1.x <- inputEvenIdx.x; r1.z <- inputEvenIdx.z + mov r1.y, r1.y // r1.y <- inputEvenIdx.y + +#line 77 + mov r3.xz, r1.xxzx // r3.x <- inputOddIdx.x; r3.z <- inputOddIdx.z + mov r3.y, r3.y // r3.y <- inputOddIdx.y + +#line 80 + nop + mov r1.xyz, r1.xyzx + +#line 29 + itof r4.y, l(0) // r4.y <- value.y + +#line 31 + imul null, r0.z, r1.x, cb0[2].x + imul null, r1.x, r1.y, cb0[2].y + iadd r0.z, r0.z, r1.x + imul null, r1.x, r1.z, cb0[2].z + iadd r0.z, r0.z, r1.x // r0.z <- indexReal + +#line 35 + ld_structured_indexable(structured_buffer, stride=4)(mixed,mixed,mixed,mixed) r4.x, r0.z, l(0), u0.xxxx // r4.x <- value.x + +#line 39 + mov r1.x, l(2) + ieq r1.x, r1.x, cb0[1].w + if_nz r1.x + +#line 40 + iadd r0.z, r0.z, cb0[2].w // r0.z <- indexImaginary + +#line 41 + ld_structured_indexable(structured_buffer, stride=4)(mixed,mixed,mixed,mixed) r4.y, r0.z, l(0), u0.xxxx + +#line 42 + endif + +#line 44 + mov r4.x, r4.x // r4.x <- .x + mov r4.y, r4.y // r4.y <- .y + +#line 80 + mov r4.xy, r4.xyxx // r4.x <- inputEvenValue.x; r4.y <- inputEvenValue.y + +#line 81 + nop + mov r3.xyz, r3.xyzx + +#line 29 + itof r1.y, l(0) // r1.y <- value.y + +#line 31 + imul null, r0.z, r3.x, cb0[2].x + imul null, r1.z, r3.y, cb0[2].y + iadd r0.z, r0.z, r1.z + imul null, r1.z, r3.z, cb0[2].z + iadd r0.z, r0.z, r1.z // r0.z <- indexReal + +#line 35 + ld_structured_indexable(structured_buffer, stride=4)(mixed,mixed,mixed,mixed) r1.x, r0.z, l(0), u0.xxxx // r1.x <- value.x + +#line 39 + mov r1.z, l(2) + ieq r1.z, r1.z, cb0[1].w + if_nz r1.z + +#line 40 + iadd r0.z, r0.z, cb0[2].w // r0.z <- indexImaginary + +#line 41 + ld_structured_indexable(structured_buffer, stride=4)(mixed,mixed,mixed,mixed) r1.y, r0.z, l(0), u0.xxxx + +#line 42 + endif + +#line 44 + mov r1.x, r1.x // r1.x <- .x + mov r1.y, r1.y // r1.y <- .y + +#line 81 + mov r1.xy, r1.xyxx // r1.x <- inputOddValue.x; r1.y <- inputOddValue.y + +#line 85 + udiv null, r0.z, r2.y, r0.w // r0.z <- k + +#line 88 + mov r1.z, l(1) + ieq r1.z, r1.z, cb0[0].w // r1.z <- isInverse + +#line 89 + movc r1.z, r1.z, l(1.000000), l(-1.000000) // r1.z <- inverseMultiplier + +#line 90 + mul r0.x, r0.x, r1.z + utof r0.z, r0.z + mul r0.x, r0.z, r0.x + utof r0.z, r0.w + div r0.x, r0.x, r0.z // r0.x <- theta + +#line 91 + sincos null, r0.z, r0.x // r0.z <- w.x + sincos r0.x, null, r0.x // r0.x <- w.y + +#line 93 + nop + mov r0.y, r0.y + +#line 21 + imul null, r2.x, r0.y, cb0[4].z // r2.x <- dftOutputIndex.x + +#line 22 + iadd r2.y, r2.x, cb0[4].w // r2.y <- dftOutputIndex.y + +#line 23 + mov r2.x, r2.x // r2.x <- .x + mov r2.y, r2.y // r2.y <- .y + +#line 93 + mov r2.xy, r2.xyxx // r2.x <- outputIndex.x; r2.y <- outputIndex.y + +#line 94 + mul r0.y, r1.x, r0.z + mul r0.w, r1.y, r0.x + mov r0.w, -r0.w + add r0.y, r0.w, r0.y + add r0.y, r0.y, r4.x + mul r0.y, r0.y, cb0[5].x + store_structured u1.x, r2.x, l(0), r0.y + +#line 95 + mul r0.y, r1.y, r0.z + mul r0.x, r1.x, r0.x + add r0.x, r0.x, r0.y + add r0.x, r0.x, r4.y + mul r0.x, r0.x, cb0[5].x + store_structured u1.x, r2.y, l(0), r0.x + +#line 96 +endif + +#line 97 +ret +// Approximately 102 instruction slots used +#endif + +const BYTE g_DFT[] = +{ + 68, 88, 66, 67, 115, 179, + 207, 128, 52, 247, 214, 247, + 41, 44, 64, 19, 139, 245, + 62, 113, 1, 0, 0, 0, + 184, 117, 0, 0, 6, 0, + 0, 0, 56, 0, 0, 0, + 104, 4, 0, 0, 120, 4, + 0, 0, 136, 4, 0, 0, + 20, 15, 0, 0, 176, 15, + 0, 0, 82, 68, 69, 70, + 40, 4, 0, 0, 3, 0, + 0, 0, 176, 0, 0, 0, + 3, 0, 0, 0, 60, 0, + 0, 0, 0, 5, 83, 67, + 5, 1, 0, 0, 0, 4, + 0, 0, 82, 68, 49, 49, + 60, 0, 0, 0, 24, 0, + 0, 0, 32, 0, 0, 0, + 40, 0, 0, 0, 36, 0, + 0, 0, 12, 0, 0, 0, + 0, 0, 0, 0, 156, 0, + 0, 0, 6, 0, 0, 0, + 6, 0, 0, 0, 1, 0, + 0, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, + 160, 0, 0, 0, 6, 0, + 0, 0, 6, 0, 0, 0, + 1, 0, 0, 0, 4, 0, + 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 1, 0, + 0, 0, 164, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 115, 114, + 99, 0, 100, 115, 116, 0, + 67, 111, 110, 115, 116, 97, + 110, 116, 115, 0, 171, 171, + 164, 0, 0, 0, 10, 0, + 0, 0, 248, 0, 0, 0, + 96, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 156, 0, 0, 0, 1, 0, + 0, 0, 128, 3, 0, 0, + 4, 0, 0, 0, 0, 0, + 0, 0, 3, 0, 0, 0, + 160, 0, 0, 0, 1, 0, + 0, 0, 216, 3, 0, 0, + 4, 0, 0, 0, 0, 0, + 0, 0, 3, 0, 0, 0, + 136, 2, 0, 0, 0, 0, + 0, 0, 4, 0, 0, 0, + 2, 0, 0, 0, 156, 2, + 0, 0, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 192, 2, + 0, 0, 4, 0, 0, 0, + 4, 0, 0, 0, 2, 0, + 0, 0, 156, 2, 0, 0, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 205, 2, 0, 0, + 8, 0, 0, 0, 4, 0, + 0, 0, 2, 0, 0, 0, + 156, 2, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 218, 2, 0, 0, 12, 0, + 0, 0, 4, 0, 0, 0, + 2, 0, 0, 0, 156, 2, + 0, 0, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 228, 2, + 0, 0, 16, 0, 0, 0, + 16, 0, 0, 0, 2, 0, + 0, 0, 248, 2, 0, 0, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 28, 3, 0, 0, + 32, 0, 0, 0, 16, 0, + 0, 0, 2, 0, 0, 0, + 248, 2, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 41, 3, 0, 0, 48, 0, + 0, 0, 16, 0, 0, 0, + 2, 0, 0, 0, 248, 2, + 0, 0, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 53, 3, + 0, 0, 64, 0, 0, 0, + 16, 0, 0, 0, 2, 0, + 0, 0, 248, 2, 0, 0, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 67, 3, 0, 0, + 80, 0, 0, 0, 4, 0, + 0, 0, 2, 0, 0, 0, + 80, 3, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 116, 3, 0, 0, 84, 0, + 0, 0, 4, 0, 0, 0, + 2, 0, 0, 0, 156, 2, + 0, 0, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 83, 116, + 97, 114, 116, 73, 110, 100, + 101, 120, 0, 100, 119, 111, + 114, 100, 0, 171, 171, 171, + 0, 0, 19, 0, 1, 0, + 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 147, 2, 0, 0, + 69, 108, 101, 109, 101, 110, + 116, 67, 111, 117, 110, 116, + 0, 68, 70, 84, 73, 116, + 101, 114, 97, 116, 105, 111, + 110, 0, 73, 115, 73, 110, + 118, 101, 114, 115, 101, 0, + 73, 110, 112, 117, 116, 83, + 105, 122, 101, 115, 0, 117, + 105, 110, 116, 52, 0, 171, + 171, 171, 1, 0, 19, 0, + 1, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 239, 2, + 0, 0, 73, 110, 112, 117, + 116, 83, 116, 114, 105, 100, + 101, 115, 0, 79, 117, 116, + 112, 117, 116, 83, 105, 122, + 101, 115, 0, 79, 117, 116, + 112, 117, 116, 83, 116, 114, + 105, 100, 101, 115, 0, 83, + 99, 97, 108, 101, 0, 102, + 108, 111, 97, 116, 0, 171, + 0, 0, 3, 0, 1, 0, + 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 73, 3, 0, 0, + 68, 70, 84, 76, 101, 110, + 103, 116, 104, 0, 171, 171, + 168, 3, 0, 0, 0, 0, + 0, 0, 4, 0, 0, 0, + 2, 0, 0, 0, 180, 3, + 0, 0, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 36, 69, + 108, 101, 109, 101, 110, 116, + 0, 171, 171, 171, 0, 0, + 3, 0, 1, 0, 1, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 73, 3, 0, 0, 168, 3, + 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 2, 0, + 0, 0, 180, 3, 0, 0, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 77, 105, 99, 114, + 111, 115, 111, 102, 116, 32, + 40, 82, 41, 32, 72, 76, + 83, 76, 32, 83, 104, 97, + 100, 101, 114, 32, 67, 111, + 109, 112, 105, 108, 101, 114, + 32, 49, 48, 46, 49, 0, + 73, 83, 71, 78, 8, 0, + 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 79, 83, + 71, 78, 8, 0, 0, 0, + 0, 0, 0, 0, 8, 0, + 0, 0, 83, 72, 69, 88, + 132, 10, 0, 0, 80, 0, + 5, 0, 161, 2, 0, 0, + 106, 136, 0, 1, 89, 0, + 0, 4, 70, 142, 32, 0, + 0, 0, 0, 0, 6, 0, + 0, 0, 158, 0, 0, 4, + 0, 224, 17, 0, 0, 0, + 0, 0, 4, 0, 0, 0, + 158, 0, 0, 4, 0, 224, + 17, 0, 1, 0, 0, 0, + 4, 0, 0, 0, 95, 0, + 0, 2, 18, 0, 2, 0, + 104, 0, 0, 2, 5, 0, + 0, 0, 155, 0, 0, 4, + 64, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, + 54, 0, 0, 5, 18, 0, + 16, 0, 0, 0, 0, 0, + 1, 64, 0, 0, 219, 15, + 201, 64, 30, 0, 0, 7, + 34, 0, 16, 0, 0, 0, + 0, 0, 10, 0, 2, 0, + 10, 128, 32, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 79, 0, 0, 8, 66, 0, + 16, 0, 0, 0, 0, 0, + 26, 0, 16, 0, 0, 0, + 0, 0, 26, 128, 32, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 31, 0, 4, 3, + 42, 0, 16, 0, 0, 0, + 0, 0, 54, 0, 0, 5, + 66, 0, 16, 0, 0, 0, + 0, 0, 1, 64, 0, 0, + 2, 0, 0, 0, 78, 0, + 0, 9, 66, 0, 16, 0, + 0, 0, 0, 0, 0, 208, + 0, 0, 26, 128, 32, 0, + 0, 0, 0, 0, 5, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 54, 0, + 0, 6, 130, 0, 16, 0, + 0, 0, 0, 0, 42, 128, + 32, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 41, 0, + 0, 7, 130, 0, 16, 0, + 0, 0, 0, 0, 1, 64, + 0, 0, 1, 0, 0, 0, + 58, 0, 16, 0, 0, 0, + 0, 0, 54, 0, 0, 5, + 18, 0, 16, 0, 1, 0, + 0, 0, 1, 64, 0, 0, + 1, 0, 0, 0, 40, 0, + 0, 5, 18, 0, 16, 0, + 1, 0, 0, 0, 10, 0, + 16, 0, 1, 0, 0, 0, + 30, 0, 0, 8, 18, 0, + 16, 0, 1, 0, 0, 0, + 10, 0, 16, 0, 1, 0, + 0, 0, 42, 128, 32, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 41, 0, 0, 7, + 18, 0, 16, 0, 1, 0, + 0, 0, 1, 64, 0, 0, + 1, 0, 0, 0, 10, 0, + 16, 0, 1, 0, 0, 0, + 58, 0, 0, 1, 54, 0, + 0, 5, 34, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 0, 0, 0, 0, + 38, 0, 0, 10, 0, 208, + 0, 0, 34, 0, 16, 0, + 1, 0, 0, 0, 42, 128, + 32, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 26, 128, + 32, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 78, 0, + 0, 8, 0, 208, 0, 0, + 34, 0, 16, 0, 1, 0, + 0, 0, 26, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 1, 0, 0, 0, + 38, 0, 0, 10, 0, 208, + 0, 0, 66, 0, 16, 0, + 1, 0, 0, 0, 42, 128, + 32, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 26, 128, + 32, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 78, 0, + 0, 8, 18, 0, 16, 0, + 2, 0, 0, 0, 0, 208, + 0, 0, 26, 0, 16, 0, + 0, 0, 0, 0, 42, 0, + 16, 0, 1, 0, 0, 0, + 78, 0, 0, 9, 34, 0, + 16, 0, 2, 0, 0, 0, + 0, 208, 0, 0, 26, 0, + 16, 0, 1, 0, 0, 0, + 42, 128, 32, 0, 0, 0, + 0, 0, 3, 0, 0, 0, + 78, 0, 0, 9, 0, 208, + 0, 0, 66, 0, 16, 0, + 2, 0, 0, 0, 26, 0, + 16, 0, 1, 0, 0, 0, + 42, 128, 32, 0, 0, 0, + 0, 0, 3, 0, 0, 0, + 54, 0, 0, 5, 18, 0, + 16, 0, 2, 0, 0, 0, + 10, 0, 16, 0, 2, 0, + 0, 0, 54, 0, 0, 5, + 34, 0, 16, 0, 2, 0, + 0, 0, 26, 0, 16, 0, + 2, 0, 0, 0, 54, 0, + 0, 5, 66, 0, 16, 0, + 2, 0, 0, 0, 42, 0, + 16, 0, 2, 0, 0, 0, + 54, 0, 0, 5, 114, 0, + 16, 0, 2, 0, 0, 0, + 70, 2, 16, 0, 2, 0, + 0, 0, 85, 0, 0, 8, + 34, 0, 16, 0, 1, 0, + 0, 0, 26, 0, 16, 0, + 2, 0, 0, 0, 42, 128, + 32, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 38, 0, + 0, 8, 0, 208, 0, 0, + 34, 0, 16, 0, 1, 0, + 0, 0, 10, 0, 16, 0, + 1, 0, 0, 0, 26, 0, + 16, 0, 1, 0, 0, 0, + 78, 0, 0, 8, 0, 208, + 0, 0, 18, 0, 16, 0, + 1, 0, 0, 0, 26, 0, + 16, 0, 2, 0, 0, 0, + 10, 0, 16, 0, 1, 0, + 0, 0, 30, 0, 0, 7, + 34, 0, 16, 0, 1, 0, + 0, 0, 10, 0, 16, 0, + 1, 0, 0, 0, 26, 0, + 16, 0, 1, 0, 0, 0, + 30, 0, 0, 7, 34, 0, + 16, 0, 3, 0, 0, 0, + 42, 0, 16, 0, 0, 0, + 0, 0, 26, 0, 16, 0, + 1, 0, 0, 0, 54, 0, + 0, 5, 82, 0, 16, 0, + 1, 0, 0, 0, 6, 2, + 16, 0, 2, 0, 0, 0, + 54, 0, 0, 5, 34, 0, + 16, 0, 1, 0, 0, 0, + 26, 0, 16, 0, 1, 0, + 0, 0, 54, 0, 0, 5, + 82, 0, 16, 0, 3, 0, + 0, 0, 6, 2, 16, 0, + 1, 0, 0, 0, 54, 0, + 0, 5, 34, 0, 16, 0, + 3, 0, 0, 0, 26, 0, + 16, 0, 3, 0, 0, 0, + 58, 0, 0, 1, 54, 0, + 0, 5, 114, 0, 16, 0, + 1, 0, 0, 0, 70, 2, + 16, 0, 1, 0, 0, 0, + 43, 0, 0, 5, 34, 0, + 16, 0, 4, 0, 0, 0, + 1, 64, 0, 0, 0, 0, + 0, 0, 38, 0, 0, 9, + 0, 208, 0, 0, 66, 0, + 16, 0, 0, 0, 0, 0, + 10, 0, 16, 0, 1, 0, + 0, 0, 10, 128, 32, 0, + 0, 0, 0, 0, 2, 0, + 0, 0, 38, 0, 0, 9, + 0, 208, 0, 0, 18, 0, + 16, 0, 1, 0, 0, 0, + 26, 0, 16, 0, 1, 0, + 0, 0, 26, 128, 32, 0, + 0, 0, 0, 0, 2, 0, + 0, 0, 30, 0, 0, 7, + 66, 0, 16, 0, 0, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 10, 0, + 16, 0, 1, 0, 0, 0, + 38, 0, 0, 9, 0, 208, + 0, 0, 18, 0, 16, 0, + 1, 0, 0, 0, 42, 0, + 16, 0, 1, 0, 0, 0, + 42, 128, 32, 0, 0, 0, + 0, 0, 2, 0, 0, 0, + 30, 0, 0, 7, 66, 0, + 16, 0, 0, 0, 0, 0, + 42, 0, 16, 0, 0, 0, + 0, 0, 10, 0, 16, 0, + 1, 0, 0, 0, 167, 0, + 0, 139, 2, 35, 0, 128, + 131, 153, 25, 0, 18, 0, + 16, 0, 4, 0, 0, 0, + 42, 0, 16, 0, 0, 0, + 0, 0, 1, 64, 0, 0, + 0, 0, 0, 0, 6, 224, + 17, 0, 0, 0, 0, 0, + 54, 0, 0, 5, 18, 0, + 16, 0, 1, 0, 0, 0, + 1, 64, 0, 0, 2, 0, + 0, 0, 32, 0, 0, 8, + 18, 0, 16, 0, 1, 0, + 0, 0, 10, 0, 16, 0, + 1, 0, 0, 0, 58, 128, + 32, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 31, 0, + 4, 3, 10, 0, 16, 0, + 1, 0, 0, 0, 30, 0, + 0, 8, 66, 0, 16, 0, + 0, 0, 0, 0, 42, 0, + 16, 0, 0, 0, 0, 0, + 58, 128, 32, 0, 0, 0, + 0, 0, 2, 0, 0, 0, + 167, 0, 0, 139, 2, 35, + 0, 128, 131, 153, 25, 0, + 34, 0, 16, 0, 4, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 1, 64, + 0, 0, 0, 0, 0, 0, + 6, 224, 17, 0, 0, 0, + 0, 0, 21, 0, 0, 1, + 54, 0, 0, 5, 18, 0, + 16, 0, 4, 0, 0, 0, + 10, 0, 16, 0, 4, 0, + 0, 0, 54, 0, 0, 5, + 34, 0, 16, 0, 4, 0, + 0, 0, 26, 0, 16, 0, + 4, 0, 0, 0, 54, 0, + 0, 5, 50, 0, 16, 0, + 4, 0, 0, 0, 70, 0, + 16, 0, 4, 0, 0, 0, + 58, 0, 0, 1, 54, 0, + 0, 5, 114, 0, 16, 0, + 3, 0, 0, 0, 70, 2, + 16, 0, 3, 0, 0, 0, + 43, 0, 0, 5, 34, 0, + 16, 0, 1, 0, 0, 0, + 1, 64, 0, 0, 0, 0, + 0, 0, 38, 0, 0, 9, + 0, 208, 0, 0, 66, 0, + 16, 0, 0, 0, 0, 0, + 10, 0, 16, 0, 3, 0, + 0, 0, 10, 128, 32, 0, + 0, 0, 0, 0, 2, 0, + 0, 0, 38, 0, 0, 9, + 0, 208, 0, 0, 66, 0, + 16, 0, 1, 0, 0, 0, + 26, 0, 16, 0, 3, 0, + 0, 0, 26, 128, 32, 0, + 0, 0, 0, 0, 2, 0, + 0, 0, 30, 0, 0, 7, + 66, 0, 16, 0, 0, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 42, 0, + 16, 0, 1, 0, 0, 0, + 38, 0, 0, 9, 0, 208, + 0, 0, 66, 0, 16, 0, + 1, 0, 0, 0, 42, 0, + 16, 0, 3, 0, 0, 0, + 42, 128, 32, 0, 0, 0, + 0, 0, 2, 0, 0, 0, + 30, 0, 0, 7, 66, 0, + 16, 0, 0, 0, 0, 0, + 42, 0, 16, 0, 0, 0, + 0, 0, 42, 0, 16, 0, + 1, 0, 0, 0, 167, 0, + 0, 139, 2, 35, 0, 128, + 131, 153, 25, 0, 18, 0, + 16, 0, 1, 0, 0, 0, + 42, 0, 16, 0, 0, 0, + 0, 0, 1, 64, 0, 0, + 0, 0, 0, 0, 6, 224, + 17, 0, 0, 0, 0, 0, + 54, 0, 0, 5, 66, 0, + 16, 0, 1, 0, 0, 0, + 1, 64, 0, 0, 2, 0, + 0, 0, 32, 0, 0, 8, + 66, 0, 16, 0, 1, 0, + 0, 0, 42, 0, 16, 0, + 1, 0, 0, 0, 58, 128, + 32, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 31, 0, + 4, 3, 42, 0, 16, 0, + 1, 0, 0, 0, 30, 0, + 0, 8, 66, 0, 16, 0, + 0, 0, 0, 0, 42, 0, + 16, 0, 0, 0, 0, 0, + 58, 128, 32, 0, 0, 0, + 0, 0, 2, 0, 0, 0, + 167, 0, 0, 139, 2, 35, + 0, 128, 131, 153, 25, 0, + 34, 0, 16, 0, 1, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 1, 64, + 0, 0, 0, 0, 0, 0, + 6, 224, 17, 0, 0, 0, + 0, 0, 21, 0, 0, 1, + 54, 0, 0, 5, 18, 0, + 16, 0, 1, 0, 0, 0, + 10, 0, 16, 0, 1, 0, + 0, 0, 54, 0, 0, 5, + 34, 0, 16, 0, 1, 0, + 0, 0, 26, 0, 16, 0, + 1, 0, 0, 0, 54, 0, + 0, 5, 50, 0, 16, 0, + 1, 0, 0, 0, 70, 0, + 16, 0, 1, 0, 0, 0, + 78, 0, 0, 8, 0, 208, + 0, 0, 66, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 2, 0, 0, 0, + 58, 0, 16, 0, 0, 0, + 0, 0, 54, 0, 0, 5, + 66, 0, 16, 0, 1, 0, + 0, 0, 1, 64, 0, 0, + 1, 0, 0, 0, 32, 0, + 0, 8, 66, 0, 16, 0, + 1, 0, 0, 0, 42, 0, + 16, 0, 1, 0, 0, 0, + 58, 128, 32, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 55, 0, 0, 9, 66, 0, + 16, 0, 1, 0, 0, 0, + 42, 0, 16, 0, 1, 0, + 0, 0, 1, 64, 0, 0, + 0, 0, 128, 63, 1, 64, + 0, 0, 0, 0, 128, 191, + 56, 0, 0, 7, 18, 0, + 16, 0, 0, 0, 0, 0, + 10, 0, 16, 0, 0, 0, + 0, 0, 42, 0, 16, 0, + 1, 0, 0, 0, 86, 0, + 0, 5, 66, 0, 16, 0, + 0, 0, 0, 0, 42, 0, + 16, 0, 0, 0, 0, 0, + 56, 0, 0, 7, 18, 0, + 16, 0, 0, 0, 0, 0, + 42, 0, 16, 0, 0, 0, + 0, 0, 10, 0, 16, 0, + 0, 0, 0, 0, 86, 0, + 0, 5, 66, 0, 16, 0, + 0, 0, 0, 0, 58, 0, + 16, 0, 0, 0, 0, 0, + 14, 0, 0, 7, 18, 0, + 16, 0, 0, 0, 0, 0, + 10, 0, 16, 0, 0, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 77, 0, + 0, 6, 0, 208, 0, 0, + 66, 0, 16, 0, 0, 0, + 0, 0, 10, 0, 16, 0, + 0, 0, 0, 0, 77, 0, + 0, 6, 18, 0, 16, 0, + 0, 0, 0, 0, 0, 208, + 0, 0, 10, 0, 16, 0, + 0, 0, 0, 0, 58, 0, + 0, 1, 54, 0, 0, 5, + 34, 0, 16, 0, 0, 0, + 0, 0, 26, 0, 16, 0, + 0, 0, 0, 0, 38, 0, + 0, 9, 0, 208, 0, 0, + 18, 0, 16, 0, 2, 0, + 0, 0, 26, 0, 16, 0, + 0, 0, 0, 0, 42, 128, + 32, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 30, 0, + 0, 8, 34, 0, 16, 0, + 2, 0, 0, 0, 10, 0, + 16, 0, 2, 0, 0, 0, + 58, 128, 32, 0, 0, 0, + 0, 0, 4, 0, 0, 0, + 54, 0, 0, 5, 18, 0, + 16, 0, 2, 0, 0, 0, + 10, 0, 16, 0, 2, 0, + 0, 0, 54, 0, 0, 5, + 34, 0, 16, 0, 2, 0, + 0, 0, 26, 0, 16, 0, + 2, 0, 0, 0, 54, 0, + 0, 5, 50, 0, 16, 0, + 2, 0, 0, 0, 70, 0, + 16, 0, 2, 0, 0, 0, + 56, 0, 0, 7, 34, 0, + 16, 0, 0, 0, 0, 0, + 10, 0, 16, 0, 1, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 56, 0, + 0, 7, 130, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 1, 0, 0, 0, + 10, 0, 16, 0, 0, 0, + 0, 0, 54, 0, 0, 6, + 130, 0, 16, 0, 0, 0, + 0, 0, 58, 0, 16, 128, + 65, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 7, + 34, 0, 16, 0, 0, 0, + 0, 0, 58, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 0, 0, 0, 0, + 0, 0, 0, 7, 34, 0, + 16, 0, 0, 0, 0, 0, + 26, 0, 16, 0, 0, 0, + 0, 0, 10, 0, 16, 0, + 4, 0, 0, 0, 56, 0, + 0, 8, 34, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 0, 0, 0, 0, + 10, 128, 32, 0, 0, 0, + 0, 0, 5, 0, 0, 0, + 168, 0, 0, 9, 18, 224, + 17, 0, 1, 0, 0, 0, + 10, 0, 16, 0, 2, 0, + 0, 0, 1, 64, 0, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 0, 0, 0, 0, + 56, 0, 0, 7, 34, 0, + 16, 0, 0, 0, 0, 0, + 26, 0, 16, 0, 1, 0, + 0, 0, 42, 0, 16, 0, + 0, 0, 0, 0, 56, 0, + 0, 7, 18, 0, 16, 0, + 0, 0, 0, 0, 10, 0, + 16, 0, 1, 0, 0, 0, + 10, 0, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 7, + 18, 0, 16, 0, 0, 0, + 0, 0, 10, 0, 16, 0, + 0, 0, 0, 0, 26, 0, + 16, 0, 0, 0, 0, 0, + 0, 0, 0, 7, 18, 0, + 16, 0, 0, 0, 0, 0, + 10, 0, 16, 0, 0, 0, + 0, 0, 26, 0, 16, 0, + 4, 0, 0, 0, 56, 0, + 0, 8, 18, 0, 16, 0, + 0, 0, 0, 0, 10, 0, + 16, 0, 0, 0, 0, 0, + 10, 128, 32, 0, 0, 0, + 0, 0, 5, 0, 0, 0, + 168, 0, 0, 9, 18, 224, + 17, 0, 1, 0, 0, 0, + 26, 0, 16, 0, 2, 0, + 0, 0, 1, 64, 0, 0, + 0, 0, 0, 0, 10, 0, + 16, 0, 0, 0, 0, 0, + 21, 0, 0, 1, 62, 0, + 0, 1, 83, 84, 65, 84, + 148, 0, 0, 0, 102, 0, + 0, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 16, 0, 0, 0, + 27, 0, 0, 0, 9, 0, + 0, 0, 1, 0, 0, 0, + 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 4, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 28, 0, 0, 0, + 1, 0, 0, 0, 4, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2, 0, + 0, 0, 83, 80, 68, 66, + 0, 102, 0, 0, 77, 105, + 99, 114, 111, 115, 111, 102, + 116, 32, 67, 47, 67, 43, + 43, 32, 77, 83, 70, 32, + 55, 46, 48, 48, 13, 10, + 26, 68, 83, 0, 0, 0, + 0, 2, 0, 0, 2, 0, + 0, 0, 51, 0, 0, 0, + 232, 0, 0, 0, 0, 0, + 0, 0, 47, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 192, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 56, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 5, 0, + 0, 0, 32, 0, 0, 0, + 60, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 6, 0, + 0, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 148, 46, 49, 1, + 68, 252, 16, 99, 1, 0, + 0, 0, 4, 192, 184, 18, + 49, 85, 209, 71, 177, 201, + 131, 74, 11, 196, 189, 67, + 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 220, 81, + 51, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 11, 202, + 49, 1, 56, 0, 0, 0, + 0, 16, 0, 0, 3, 16, + 0, 0, 88, 0, 0, 0, + 11, 0, 255, 255, 4, 0, + 0, 0, 255, 255, 3, 0, + 0, 0, 0, 0, 12, 0, + 0, 0, 12, 0, 0, 0, + 8, 0, 0, 0, 20, 0, + 0, 0, 0, 0, 0, 0, + 26, 0, 1, 22, 0, 0, + 0, 0, 9, 16, 0, 0, + 68, 101, 99, 111, 109, 112, + 111, 115, 101, 73, 110, 100, + 101, 120, 0, 241, 26, 0, + 1, 22, 0, 0, 0, 0, + 11, 16, 0, 0, 82, 101, + 97, 100, 83, 111, 117, 114, + 99, 101, 86, 97, 108, 117, + 101, 0, 30, 0, 1, 22, + 0, 0, 0, 0, 13, 16, + 0, 0, 67, 111, 109, 112, + 117, 116, 101, 68, 101, 115, + 116, 73, 110, 100, 101, 120, + 0, 243, 242, 241, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 102, 141, 2, 0, 25, 96, + 3, 0, 80, 133, 1, 0, + 187, 122, 1, 0, 63, 252, + 2, 0, 117, 131, 1, 0, + 146, 230, 3, 0, 122, 191, + 0, 0, 65, 36, 1, 0, + 41, 153, 1, 0, 109, 24, + 1, 0, 24, 140, 1, 0, + 221, 8, 3, 0, 143, 195, + 3, 0, 17, 248, 0, 0, + 53, 174, 3, 0, 214, 6, + 3, 0, 98, 163, 2, 0, + 169, 232, 3, 0, 62, 3, + 3, 0, 220, 192, 1, 0, + 9, 241, 2, 0, 0, 16, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 82, 87, 83, 116, + 114, 117, 99, 116, 117, 114, + 101, 100, 66, 117, 102, 102, + 101, 114, 60, 102, 108, 111, + 97, 116, 62, 32, 115, 114, + 99, 32, 58, 32, 114, 101, + 103, 105, 115, 116, 101, 114, + 40, 117, 48, 41, 59, 13, + 10, 82, 87, 83, 116, 114, + 117, 99, 116, 117, 114, 101, + 100, 66, 117, 102, 102, 101, + 114, 60, 102, 108, 111, 97, + 116, 62, 32, 100, 115, 116, + 32, 58, 32, 114, 101, 103, + 105, 115, 116, 101, 114, 40, + 117, 49, 41, 59, 13, 10, + 13, 10, 99, 98, 117, 102, + 102, 101, 114, 32, 67, 111, + 110, 115, 116, 97, 110, 116, + 115, 13, 10, 123, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 32, 83, 116, 97, + 114, 116, 73, 110, 100, 101, + 120, 59, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 69, 108, 101, 109, 101, + 110, 116, 67, 111, 117, 110, + 116, 59, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 68, 70, 84, 73, 116, + 101, 114, 97, 116, 105, 111, + 110, 59, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 73, 115, 73, 110, 118, + 101, 114, 115, 101, 59, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 52, 32, 73, + 110, 112, 117, 116, 83, 105, + 122, 101, 115, 59, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 52, 32, 73, 110, + 112, 117, 116, 83, 116, 114, + 105, 100, 101, 115, 59, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 52, 32, 79, + 117, 116, 112, 117, 116, 83, + 105, 122, 101, 115, 59, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 52, 32, 79, + 117, 116, 112, 117, 116, 83, + 116, 114, 105, 100, 101, 115, + 59, 13, 10, 32, 32, 32, + 32, 102, 108, 111, 97, 116, + 32, 83, 99, 97, 108, 101, + 59, 13, 10, 32, 32, 32, + 32, 117, 105, 110, 116, 32, + 68, 70, 84, 76, 101, 110, + 103, 116, 104, 59, 13, 10, + 125, 59, 13, 10, 13, 10, + 47, 47, 32, 82, 101, 116, + 117, 114, 110, 115, 32, 116, + 104, 101, 32, 105, 110, 100, + 105, 99, 101, 115, 32, 102, + 111, 114, 32, 116, 104, 101, + 32, 114, 101, 97, 108, 32, + 97, 110, 100, 32, 99, 111, + 109, 112, 108, 101, 120, 32, + 111, 117, 116, 112, 117, 116, + 32, 117, 97, 118, 13, 10, + 117, 105, 110, 116, 50, 32, + 67, 111, 109, 112, 117, 116, + 101, 68, 101, 115, 116, 73, + 110, 100, 101, 120, 40, 117, + 105, 110, 116, 32, 105, 110, + 100, 101, 120, 41, 13, 10, + 123, 13, 10, 32, 32, 32, + 32, 117, 105, 110, 116, 50, + 32, 100, 102, 116, 79, 117, + 116, 112, 117, 116, 73, 110, + 100, 101, 120, 32, 61, 32, + 117, 105, 110, 116, 50, 40, + 105, 110, 100, 101, 120, 32, + 42, 32, 79, 117, 116, 112, + 117, 116, 83, 116, 114, 105, + 100, 101, 115, 91, 50, 93, + 44, 32, 48, 41, 59, 13, + 10, 32, 32, 32, 32, 100, + 102, 116, 79, 117, 116, 112, + 117, 116, 73, 110, 100, 101, + 120, 46, 121, 32, 61, 32, + 100, 102, 116, 79, 117, 116, + 112, 117, 116, 73, 110, 100, + 101, 120, 46, 120, 32, 43, + 32, 79, 117, 116, 112, 117, + 116, 83, 116, 114, 105, 100, + 101, 115, 91, 51, 93, 59, + 13, 10, 32, 32, 32, 32, + 114, 101, 116, 117, 114, 110, + 32, 100, 102, 116, 79, 117, + 116, 112, 117, 116, 73, 110, + 100, 101, 120, 59, 13, 10, + 125, 13, 10, 13, 10, 47, + 47, 32, 84, 104, 101, 32, + 114, 101, 116, 117, 114, 110, + 101, 100, 32, 118, 97, 108, + 117, 101, 32, 105, 115, 32, + 102, 108, 111, 97, 116, 50, + 44, 32, 99, 111, 114, 114, + 101, 115, 112, 111, 110, 100, + 105, 110, 103, 32, 116, 111, + 32, 116, 104, 101, 32, 99, + 111, 109, 112, 108, 101, 120, + 32, 110, 117, 109, 98, 101, + 114, 32, 97, 116, 32, 116, + 104, 101, 32, 105, 110, 100, + 101, 120, 13, 10, 102, 108, + 111, 97, 116, 50, 32, 82, + 101, 97, 100, 83, 111, 117, + 114, 99, 101, 86, 97, 108, + 117, 101, 40, 117, 105, 110, + 116, 51, 32, 105, 110, 100, + 101, 120, 41, 13, 10, 123, + 13, 10, 32, 32, 32, 32, + 102, 108, 111, 97, 116, 50, + 32, 118, 97, 108, 117, 101, + 32, 61, 32, 102, 108, 111, + 97, 116, 50, 40, 48, 44, + 32, 48, 41, 59, 13, 10, + 13, 10, 32, 32, 32, 32, + 117, 105, 110, 116, 32, 105, + 110, 100, 101, 120, 82, 101, + 97, 108, 32, 61, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 105, 110, 100, 101, + 120, 46, 120, 32, 42, 32, + 73, 110, 112, 117, 116, 83, + 116, 114, 105, 100, 101, 115, + 91, 48, 93, 32, 43, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 105, 110, 100, + 101, 120, 46, 121, 32, 42, + 32, 73, 110, 112, 117, 116, + 83, 116, 114, 105, 100, 101, + 115, 91, 49, 93, 32, 43, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 105, 110, + 100, 101, 120, 46, 122, 32, + 42, 32, 73, 110, 112, 117, + 116, 83, 116, 114, 105, 100, + 101, 115, 91, 50, 93, 59, + 13, 10, 32, 32, 32, 32, + 118, 97, 108, 117, 101, 46, + 120, 32, 61, 32, 115, 114, + 99, 91, 105, 110, 100, 101, + 120, 82, 101, 97, 108, 93, + 59, 13, 10, 13, 10, 32, + 32, 32, 32, 47, 47, 32, + 73, 102, 32, 114, 101, 97, + 108, 32, 118, 97, 108, 117, + 101, 100, 44, 32, 118, 97, + 108, 117, 101, 46, 121, 32, + 105, 115, 32, 100, 101, 102, + 97, 117, 108, 116, 101, 100, + 32, 116, 111, 32, 48, 13, + 10, 32, 32, 32, 32, 47, + 47, 32, 73, 102, 32, 99, + 111, 109, 112, 108, 101, 120, + 32, 118, 97, 108, 117, 101, + 100, 32, 105, 110, 112, 117, + 116, 44, 32, 97, 115, 115, + 105, 103, 110, 32, 116, 104, + 101, 32, 99, 111, 109, 112, + 108, 101, 120, 32, 112, 97, + 114, 116, 32, 116, 111, 32, + 110, 111, 110, 45, 122, 101, + 114, 111, 46, 46, 46, 13, + 10, 32, 32, 32, 32, 105, + 102, 32, 40, 73, 110, 112, + 117, 116, 83, 105, 122, 101, + 115, 91, 51, 93, 32, 61, + 61, 32, 50, 41, 32, 123, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 117, 105, + 110, 116, 32, 105, 110, 100, + 101, 120, 73, 109, 97, 103, + 105, 110, 97, 114, 121, 32, + 61, 32, 105, 110, 100, 101, + 120, 82, 101, 97, 108, 32, + 43, 32, 73, 110, 112, 117, + 116, 83, 116, 114, 105, 100, + 101, 115, 91, 51, 93, 59, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 118, 97, + 108, 117, 101, 46, 121, 32, + 61, 32, 115, 114, 99, 91, + 105, 110, 100, 101, 120, 73, + 109, 97, 103, 105, 110, 97, + 114, 121, 93, 59, 13, 10, + 32, 32, 32, 32, 125, 13, + 10, 13, 10, 32, 32, 32, + 32, 114, 101, 116, 117, 114, + 110, 32, 118, 97, 108, 117, + 101, 59, 13, 10, 125, 13, + 10, 13, 10, 117, 105, 110, + 116, 51, 32, 68, 101, 99, + 111, 109, 112, 111, 115, 101, + 73, 110, 100, 101, 120, 40, + 117, 105, 110, 116, 32, 105, + 110, 100, 101, 120, 41, 13, + 10, 123, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 116, 101, 109, 112, 32, + 61, 32, 105, 110, 100, 101, + 120, 32, 37, 32, 40, 79, + 117, 116, 112, 117, 116, 83, + 105, 122, 101, 115, 91, 49, + 93, 32, 42, 32, 79, 117, + 116, 112, 117, 116, 83, 105, + 122, 101, 115, 91, 50, 93, + 41, 59, 13, 10, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 51, 32, 105, 100, + 120, 32, 61, 32, 117, 105, + 110, 116, 51, 40, 48, 44, + 32, 48, 44, 32, 48, 41, + 59, 13, 10, 32, 32, 32, + 32, 105, 100, 120, 46, 120, + 32, 61, 32, 105, 110, 100, + 101, 120, 32, 47, 32, 40, + 79, 117, 116, 112, 117, 116, + 83, 105, 122, 101, 115, 91, + 49, 93, 32, 42, 32, 79, + 117, 116, 112, 117, 116, 83, + 105, 122, 101, 115, 91, 50, + 93, 41, 59, 13, 10, 32, + 32, 32, 32, 105, 100, 120, + 46, 121, 32, 61, 32, 116, + 101, 109, 112, 32, 47, 32, + 79, 117, 116, 112, 117, 116, + 83, 105, 122, 101, 115, 91, + 50, 93, 59, 32, 47, 47, + 32, 84, 104, 105, 115, 32, + 99, 111, 114, 114, 101, 115, + 112, 111, 110, 100, 115, 32, + 116, 111, 32, 116, 104, 101, + 32, 115, 49, 39, 116, 104, + 32, 101, 108, 101, 109, 101, + 110, 116, 32, 111, 102, 32, + 116, 104, 101, 32, 100, 102, + 116, 13, 10, 32, 32, 32, + 32, 105, 100, 120, 46, 122, + 32, 61, 32, 116, 101, 109, + 112, 32, 37, 32, 79, 117, + 116, 112, 117, 116, 83, 105, + 122, 101, 115, 91, 50, 93, + 59, 13, 10, 32, 32, 32, + 32, 114, 101, 116, 117, 114, + 110, 32, 105, 100, 120, 59, + 13, 10, 125, 13, 10, 13, + 10, 91, 110, 117, 109, 116, + 104, 114, 101, 97, 100, 115, + 40, 54, 52, 44, 32, 49, + 44, 32, 49, 41, 93, 13, + 10, 118, 111, 105, 100, 32, + 68, 70, 84, 40, 117, 105, + 110, 116, 51, 32, 100, 116, + 105, 100, 32, 58, 32, 83, + 86, 95, 68, 105, 115, 112, + 97, 116, 99, 104, 84, 104, + 114, 101, 97, 100, 73, 100, + 41, 13, 10, 123, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 32, 105, 110, 100, + 101, 120, 32, 61, 32, 83, + 116, 97, 114, 116, 73, 110, + 100, 101, 120, 32, 43, 32, + 100, 116, 105, 100, 46, 120, + 59, 13, 10, 32, 32, 32, + 32, 105, 102, 32, 40, 105, + 110, 100, 101, 120, 32, 60, + 32, 69, 108, 101, 109, 101, + 110, 116, 67, 111, 117, 110, + 116, 41, 13, 10, 32, 32, + 32, 32, 123, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 117, 105, 110, 116, 32, + 104, 97, 108, 102, 84, 111, + 116, 97, 108, 68, 70, 84, + 76, 101, 110, 103, 116, 104, + 32, 61, 32, 68, 70, 84, + 76, 101, 110, 103, 116, 104, + 32, 47, 32, 50, 59, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 117, 105, 110, + 116, 32, 78, 32, 61, 32, + 49, 32, 60, 60, 32, 68, + 70, 84, 73, 116, 101, 114, + 97, 116, 105, 111, 110, 59, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 117, 105, + 110, 116, 32, 104, 97, 108, + 102, 78, 32, 61, 32, 49, + 32, 60, 60, 32, 40, 68, + 70, 84, 73, 116, 101, 114, + 97, 116, 105, 111, 110, 32, + 45, 32, 49, 41, 59, 13, + 10, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 47, + 47, 32, 71, 101, 116, 32, + 105, 110, 112, 117, 116, 32, + 101, 118, 101, 110, 32, 97, + 110, 100, 32, 111, 100, 100, + 32, 105, 110, 100, 105, 99, + 101, 115, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 47, 47, 32, 68, 101, 99, + 111, 109, 112, 111, 115, 101, + 32, 116, 104, 101, 32, 99, + 117, 114, 114, 101, 110, 116, + 32, 105, 110, 100, 101, 120, + 32, 105, 110, 116, 111, 32, + 105, 116, 115, 32, 108, 111, + 99, 97, 116, 105, 111, 110, + 32, 105, 110, 32, 116, 104, + 101, 32, 112, 97, 99, 107, + 101, 100, 32, 116, 101, 110, + 115, 111, 114, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 117, 105, 110, 116, 50, + 32, 105, 110, 112, 117, 116, + 69, 118, 101, 110, 79, 100, + 100, 73, 110, 100, 101, 120, + 80, 97, 105, 114, 32, 61, + 32, 117, 105, 110, 116, 50, + 40, 48, 44, 32, 48, 41, + 59, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 117, + 105, 110, 116, 51, 32, 105, + 100, 120, 32, 61, 32, 68, + 101, 99, 111, 109, 112, 111, + 115, 101, 73, 110, 100, 101, + 120, 40, 105, 110, 100, 101, + 120, 41, 59, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 105, 110, 112, 117, 116, + 69, 118, 101, 110, 79, 100, + 100, 73, 110, 100, 101, 120, + 80, 97, 105, 114, 46, 120, + 32, 61, 32, 40, 105, 100, + 120, 46, 121, 32, 62, 62, + 32, 68, 70, 84, 73, 116, + 101, 114, 97, 116, 105, 111, + 110, 41, 32, 42, 32, 104, + 97, 108, 102, 78, 32, 43, + 32, 40, 105, 100, 120, 46, + 121, 32, 37, 32, 104, 97, + 108, 102, 78, 41, 59, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 105, 110, 112, + 117, 116, 69, 118, 101, 110, + 79, 100, 100, 73, 110, 100, + 101, 120, 80, 97, 105, 114, + 46, 121, 32, 61, 32, 105, + 110, 112, 117, 116, 69, 118, + 101, 110, 79, 100, 100, 73, + 110, 100, 101, 120, 80, 97, + 105, 114, 46, 120, 32, 43, + 32, 104, 97, 108, 102, 84, + 111, 116, 97, 108, 68, 70, + 84, 76, 101, 110, 103, 116, + 104, 59, 13, 10, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 47, 47, 32, 67, + 114, 101, 97, 116, 101, 32, + 102, 117, 108, 108, 32, 105, + 110, 100, 101, 120, 32, 102, + 111, 114, 32, 101, 118, 101, + 110, 32, 97, 110, 100, 32, + 111, 100, 100, 32, 118, 97, + 108, 117, 101, 115, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 117, 105, 110, 116, + 51, 32, 105, 110, 112, 117, + 116, 69, 118, 101, 110, 73, + 100, 120, 32, 61, 32, 117, + 105, 110, 116, 51, 40, 105, + 100, 120, 46, 120, 44, 32, + 105, 110, 112, 117, 116, 69, + 118, 101, 110, 79, 100, 100, + 73, 110, 100, 101, 120, 80, + 97, 105, 114, 46, 120, 44, + 32, 105, 100, 120, 46, 122, + 41, 59, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 117, 105, 110, 116, 51, 32, + 105, 110, 112, 117, 116, 79, + 100, 100, 73, 100, 120, 32, + 61, 32, 117, 105, 110, 116, + 51, 40, 105, 100, 120, 46, + 120, 44, 32, 105, 110, 112, + 117, 116, 69, 118, 101, 110, + 79, 100, 100, 73, 110, 100, + 101, 120, 80, 97, 105, 114, + 46, 121, 44, 32, 105, 100, + 120, 46, 122, 41, 59, 13, + 10, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 47, + 47, 32, 82, 101, 97, 100, + 32, 105, 110, 112, 117, 116, + 32, 101, 118, 101, 110, 32, + 97, 110, 100, 32, 111, 100, + 100, 32, 118, 97, 108, 117, + 101, 115, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 102, 108, 111, 97, 116, 50, + 32, 105, 110, 112, 117, 116, + 69, 118, 101, 110, 86, 97, + 108, 117, 101, 32, 61, 32, + 82, 101, 97, 100, 83, 111, + 117, 114, 99, 101, 86, 97, + 108, 117, 101, 40, 105, 110, + 112, 117, 116, 69, 118, 101, + 110, 73, 100, 120, 41, 59, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 102, 108, + 111, 97, 116, 50, 32, 105, + 110, 112, 117, 116, 79, 100, + 100, 86, 97, 108, 117, 101, + 32, 61, 32, 82, 101, 97, + 100, 83, 111, 117, 114, 99, + 101, 86, 97, 108, 117, 101, + 40, 105, 110, 112, 117, 116, + 79, 100, 100, 73, 100, 120, + 41, 59, 13, 10, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 47, 47, 32, 67, + 114, 101, 97, 116, 101, 32, + 99, 111, 101, 102, 102, 105, + 99, 105, 101, 110, 116, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 47, 47, 32, + 119, 40, 107, 44, 32, 78, + 41, 32, 61, 32, 101, 94, + 40, 105, 42, 50, 42, 112, + 105, 32, 42, 32, 107, 32, + 47, 32, 78, 41, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 107, 32, 61, 32, 105, + 100, 120, 46, 121, 32, 37, + 32, 78, 59, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 115, 116, 97, 116, 105, + 99, 32, 99, 111, 110, 115, + 116, 32, 102, 108, 111, 97, + 116, 32, 80, 73, 32, 61, + 32, 51, 46, 49, 52, 49, + 53, 57, 50, 54, 53, 102, + 59, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 115, + 116, 97, 116, 105, 99, 32, + 99, 111, 110, 115, 116, 32, + 102, 108, 111, 97, 116, 32, + 84, 65, 85, 32, 61, 32, + 80, 73, 32, 42, 32, 50, + 59, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 98, + 111, 111, 108, 32, 105, 115, + 73, 110, 118, 101, 114, 115, + 101, 32, 61, 32, 73, 115, + 73, 110, 118, 101, 114, 115, + 101, 32, 61, 61, 32, 49, + 59, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 99, + 111, 110, 115, 116, 32, 102, + 108, 111, 97, 116, 32, 105, + 110, 118, 101, 114, 115, 101, + 77, 117, 108, 116, 105, 112, + 108, 105, 101, 114, 32, 61, + 32, 105, 115, 73, 110, 118, + 101, 114, 115, 101, 32, 63, + 32, 49, 46, 102, 32, 58, + 32, 45, 49, 46, 102, 59, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 102, 108, + 111, 97, 116, 32, 116, 104, + 101, 116, 97, 32, 61, 32, + 105, 110, 118, 101, 114, 115, + 101, 77, 117, 108, 116, 105, + 112, 108, 105, 101, 114, 32, + 42, 32, 84, 65, 85, 32, + 42, 32, 40, 102, 108, 111, + 97, 116, 41, 107, 32, 47, + 32, 40, 102, 108, 111, 97, + 116, 41, 78, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 102, 108, 111, 97, + 116, 50, 32, 119, 32, 61, + 32, 102, 108, 111, 97, 116, + 50, 40, 99, 111, 115, 40, + 116, 104, 101, 116, 97, 41, + 44, 32, 115, 105, 110, 40, + 116, 104, 101, 116, 97, 41, + 41, 59, 13, 10, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 117, 105, 110, 116, + 50, 32, 111, 117, 116, 112, + 117, 116, 73, 110, 100, 101, + 120, 32, 61, 32, 67, 111, + 109, 112, 117, 116, 101, 68, + 101, 115, 116, 73, 110, 100, + 101, 120, 40, 105, 110, 100, + 101, 120, 41, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 100, 115, 116, 91, + 111, 117, 116, 112, 117, 116, + 73, 110, 100, 101, 120, 46, + 120, 93, 32, 61, 32, 83, + 99, 97, 108, 101, 32, 42, + 32, 40, 105, 110, 112, 117, + 116, 69, 118, 101, 110, 86, + 97, 108, 117, 101, 46, 120, + 32, 43, 32, 40, 119, 46, + 120, 32, 42, 32, 105, 110, + 112, 117, 116, 79, 100, 100, + 86, 97, 108, 117, 101, 46, + 120, 32, 45, 32, 119, 46, + 121, 32, 42, 32, 105, 110, + 112, 117, 116, 79, 100, 100, + 86, 97, 108, 117, 101, 46, + 121, 41, 41, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 100, 115, 116, 91, + 111, 117, 116, 112, 117, 116, + 73, 110, 100, 101, 120, 46, + 121, 93, 32, 61, 32, 83, + 99, 97, 108, 101, 32, 42, + 32, 40, 105, 110, 112, 117, + 116, 69, 118, 101, 110, 86, + 97, 108, 117, 101, 46, 121, + 32, 43, 32, 40, 119, 46, + 120, 32, 42, 32, 105, 110, + 112, 117, 116, 79, 100, 100, + 86, 97, 108, 117, 101, 46, + 121, 32, 43, 32, 119, 46, + 121, 32, 42, 32, 105, 110, + 112, 117, 116, 79, 100, 100, + 86, 97, 108, 117, 101, 46, + 120, 41, 41, 59, 13, 10, + 32, 32, 32, 32, 125, 13, + 10, 125, 13, 10, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 254, 239, + 254, 239, 1, 0, 0, 0, + 195, 13, 0, 0, 0, 69, + 58, 92, 119, 111, 114, 107, + 92, 111, 110, 110, 120, 114, + 117, 110, 116, 105, 109, 101, + 51, 92, 111, 110, 110, 120, + 114, 117, 110, 116, 105, 109, + 101, 92, 99, 111, 114, 101, + 92, 112, 114, 111, 118, 105, + 100, 101, 114, 115, 92, 100, + 109, 108, 92, 68, 109, 108, + 69, 120, 101, 99, 117, 116, + 105, 111, 110, 80, 114, 111, + 118, 105, 100, 101, 114, 92, + 115, 114, 99, 92, 79, 112, + 101, 114, 97, 116, 111, 114, + 115, 92, 83, 104, 97, 100, + 101, 114, 115, 92, 115, 116, + 111, 99, 107, 104, 97, 109, + 46, 104, 108, 115, 108, 0, + 0, 101, 58, 92, 119, 111, + 114, 107, 92, 111, 110, 110, + 120, 114, 117, 110, 116, 105, + 109, 101, 51, 92, 111, 110, + 110, 120, 114, 117, 110, 116, + 105, 109, 101, 92, 99, 111, + 114, 101, 92, 112, 114, 111, + 118, 105, 100, 101, 114, 115, + 92, 100, 109, 108, 92, 100, + 109, 108, 101, 120, 101, 99, + 117, 116, 105, 111, 110, 112, + 114, 111, 118, 105, 100, 101, + 114, 92, 115, 114, 99, 92, + 111, 112, 101, 114, 97, 116, + 111, 114, 115, 92, 115, 104, + 97, 100, 101, 114, 115, 92, + 115, 116, 111, 99, 107, 104, + 97, 109, 46, 104, 108, 115, + 108, 0, 82, 87, 83, 116, + 114, 117, 99, 116, 117, 114, + 101, 100, 66, 117, 102, 102, + 101, 114, 60, 102, 108, 111, + 97, 116, 62, 32, 115, 114, + 99, 32, 58, 32, 114, 101, + 103, 105, 115, 116, 101, 114, + 40, 117, 48, 41, 59, 13, + 10, 82, 87, 83, 116, 114, + 117, 99, 116, 117, 114, 101, + 100, 66, 117, 102, 102, 101, + 114, 60, 102, 108, 111, 97, + 116, 62, 32, 100, 115, 116, + 32, 58, 32, 114, 101, 103, + 105, 115, 116, 101, 114, 40, + 117, 49, 41, 59, 13, 10, + 13, 10, 99, 98, 117, 102, + 102, 101, 114, 32, 67, 111, + 110, 115, 116, 97, 110, 116, + 115, 13, 10, 123, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 32, 83, 116, 97, + 114, 116, 73, 110, 100, 101, + 120, 59, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 69, 108, 101, 109, 101, + 110, 116, 67, 111, 117, 110, + 116, 59, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 68, 70, 84, 73, 116, + 101, 114, 97, 116, 105, 111, + 110, 59, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 73, 115, 73, 110, 118, + 101, 114, 115, 101, 59, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 52, 32, 73, + 110, 112, 117, 116, 83, 105, + 122, 101, 115, 59, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 52, 32, 73, 110, + 112, 117, 116, 83, 116, 114, + 105, 100, 101, 115, 59, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 52, 32, 79, + 117, 116, 112, 117, 116, 83, + 105, 122, 101, 115, 59, 13, + 27, 226, 48, 1, 128, 0, + 0, 0, 126, 155, 20, 26, + 50, 190, 216, 1, 1, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 2, 0, 0, 0, + 1, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, + 111, 0, 0, 0, 40, 0, + 0, 0, 27, 226, 48, 1, + 45, 40, 252, 62, 230, 12, + 0, 0, 1, 0, 0, 0, + 110, 0, 0, 0, 111, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 4, 0, 0, 0, + 66, 0, 60, 17, 16, 1, + 0, 0, 0, 1, 10, 0, + 1, 0, 1, 0, 93, 88, + 10, 0, 1, 0, 1, 0, + 93, 88, 77, 105, 99, 114, + 111, 115, 111, 102, 116, 32, + 40, 82, 41, 32, 72, 76, + 83, 76, 32, 83, 104, 97, + 100, 101, 114, 32, 67, 111, + 109, 112, 105, 108, 101, 114, + 32, 49, 48, 46, 49, 0, + 0, 0, 50, 0, 61, 17, + 1, 104, 108, 115, 108, 70, + 108, 97, 103, 115, 0, 48, + 120, 53, 0, 104, 108, 115, + 108, 84, 97, 114, 103, 101, + 116, 0, 99, 115, 95, 53, + 95, 48, 0, 104, 108, 115, + 108, 69, 110, 116, 114, 121, + 0, 68, 70, 84, 0, 0, + 42, 0, 16, 17, 0, 0, + 0, 0, 20, 16, 0, 0, + 0, 0, 0, 0, 40, 10, + 0, 0, 0, 0, 0, 0, + 40, 10, 0, 0, 3, 16, + 0, 0, 92, 0, 0, 0, + 1, 0, 160, 68, 70, 84, + 0, 0, 42, 0, 62, 17, + 0, 16, 0, 0, 9, 0, + 100, 116, 105, 100, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 32, 0, 5, 0, + 0, 0, 4, 0, 92, 0, + 0, 0, 1, 0, 40, 10, + 176, 255, 255, 255, 22, 0, + 80, 17, 32, 0, 5, 0, + 4, 0, 4, 0, 92, 0, + 0, 0, 1, 0, 40, 10, + 180, 255, 255, 255, 22, 0, + 80, 17, 32, 0, 5, 0, + 8, 0, 4, 0, 92, 0, + 0, 0, 1, 0, 40, 10, + 184, 255, 255, 255, 46, 0, + 62, 17, 117, 0, 0, 0, + 0, 0, 105, 110, 100, 101, + 120, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 1, 0, + 0, 0, 4, 0, 140, 0, + 0, 0, 1, 0, 140, 8, + 4, 0, 0, 0, 58, 0, + 62, 17, 117, 0, 0, 0, + 0, 0, 104, 97, 108, 102, + 84, 111, 116, 97, 108, 68, + 70, 84, 76, 101, 110, 103, + 116, 104, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 1, 0, + 0, 0, 4, 0, 240, 0, + 0, 0, 1, 0, 16, 3, + 8, 0, 0, 0, 42, 0, + 62, 17, 117, 0, 0, 0, + 0, 0, 78, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 1, 0, 0, 0, 4, 0, + 36, 1, 0, 0, 1, 0, + 16, 8, 12, 0, 0, 0, + 46, 0, 62, 17, 117, 0, + 0, 0, 0, 0, 104, 97, + 108, 102, 78, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 1, 0, 0, 0, 4, 0, + 136, 1, 0, 0, 1, 0, + 160, 1, 16, 0, 0, 0, + 62, 0, 62, 17, 4, 16, + 0, 0, 8, 0, 105, 110, + 112, 117, 116, 69, 118, 101, + 110, 79, 100, 100, 73, 110, + 100, 101, 120, 80, 97, 105, + 114, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 0, 0, 4, 0, 68, 3, + 0, 0, 1, 0, 96, 2, + 20, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 4, 0, 4, 0, 96, 3, + 0, 0, 1, 0, 32, 7, + 52, 0, 0, 0, 42, 0, + 62, 17, 0, 16, 0, 0, + 8, 0, 105, 100, 120, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 5, 0, 0, 0, 4, 0, + 200, 2, 0, 0, 1, 0, + 216, 5, 32, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 5, 0, 4, 0, 4, 0, + 200, 2, 0, 0, 1, 0, + 248, 5, 36, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 5, 0, 8, 0, 4, 0, + 200, 2, 0, 0, 1, 0, + 184, 7, 40, 0, 0, 0, + 50, 0, 62, 17, 0, 16, + 0, 0, 8, 0, 105, 110, + 112, 117, 116, 69, 118, 101, + 110, 73, 100, 120, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 0, 0, 4, 0, 116, 3, + 0, 0, 1, 0, 176, 0, + 16, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 8, 0, 4, 0, 116, 3, + 0, 0, 1, 0, 120, 2, + 24, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 4, 0, 4, 0, 136, 3, + 0, 0, 1, 0, 28, 2, + 20, 0, 0, 0, 50, 0, + 62, 17, 0, 16, 0, 0, + 8, 0, 105, 110, 112, 117, + 116, 79, 100, 100, 73, 100, + 120, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 156, 3, 0, 0, + 1, 0, 228, 6, 48, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 8, 0, + 4, 0, 156, 3, 0, 0, + 1, 0, 228, 6, 56, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 176, 3, 0, 0, + 1, 0, 208, 6, 52, 0, + 0, 0, 54, 0, 62, 17, + 5, 16, 0, 0, 8, 0, + 105, 110, 112, 117, 116, 69, + 118, 101, 110, 86, 97, 108, + 117, 101, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 0, 0, 4, 0, 120, 5, + 0, 0, 1, 0, 8, 5, + 64, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 4, 0, 4, 0, 120, 5, + 0, 0, 1, 0, 8, 5, + 68, 0, 0, 0, 54, 0, + 62, 17, 5, 16, 0, 0, + 8, 0, 105, 110, 112, 117, + 116, 79, 100, 100, 86, 97, + 108, 117, 101, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 5, 0, 0, 0, 4, 0, + 64, 7, 0, 0, 1, 0, + 64, 3, 16, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 5, 0, 4, 0, 4, 0, + 64, 7, 0, 0, 1, 0, + 64, 3, 20, 0, 0, 0, + 42, 0, 62, 17, 117, 0, + 0, 0, 0, 0, 107, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 96, 7, 0, 0, + 1, 0, 136, 0, 8, 0, + 0, 0, 42, 0, 62, 17, + 6, 16, 0, 0, 0, 0, + 84, 65, 85, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 1, 0, + 0, 0, 4, 0, 112, 0, + 0, 0, 1, 0, 100, 7, + 0, 0, 0, 0, 50, 0, + 62, 17, 98, 0, 0, 0, + 0, 0, 105, 115, 73, 110, + 118, 101, 114, 115, 101, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 148, 7, 0, 0, + 1, 0, 36, 0, 24, 0, + 0, 0, 58, 0, 62, 17, + 6, 16, 0, 0, 0, 0, + 105, 110, 118, 101, 114, 115, + 101, 77, 117, 108, 116, 105, + 112, 108, 105, 101, 114, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 184, 7, 0, 0, + 1, 0, 200, 2, 24, 0, + 0, 0, 46, 0, 62, 17, + 64, 0, 0, 0, 0, 0, + 116, 104, 101, 116, 97, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 52, 8, 0, 0, + 1, 0, 48, 0, 0, 0, + 0, 0, 42, 0, 62, 17, + 5, 16, 0, 0, 8, 0, + 119, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 0, 0, 4, 0, 76, 8, + 0, 0, 1, 0, 52, 2, + 8, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 4, 0, 4, 0, 100, 8, + 0, 0, 1, 0, 156, 1, + 0, 0, 0, 0, 50, 0, + 62, 17, 4, 16, 0, 0, + 8, 0, 111, 117, 116, 112, + 117, 116, 73, 110, 100, 101, + 120, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 252, 8, 0, 0, + 1, 0, 132, 1, 32, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 252, 8, 0, 0, + 1, 0, 132, 1, 36, 0, + 0, 0, 102, 0, 77, 17, + 124, 0, 0, 0, 24, 9, + 0, 0, 0, 16, 0, 0, + 7, 0, 9, 5, 13, 58, + 6, 2, 3, 129, 68, 13, + 54, 6, 6, 3, 72, 13, + 34, 6, 2, 3, 72, 6, + 2, 3, 36, 13, 15, 6, + 2, 12, 60, 36, 8, 0, + 9, 26, 13, 56, 1, 129, + 160, 6, 13, 3, 0, 9, + 17, 13, 57, 3, 40, 9, + 22, 13, 52, 6, 6, 3, + 32, 9, 13, 13, 53, 3, + 40, 13, 33, 6, 2, 3, + 32, 6, 2, 3, 36, 9, + 5, 13, 15, 6, 2, 12, + 60, 36, 0, 0, 70, 0, + 62, 17, 8, 16, 0, 0, + 136, 0, 60, 68, 101, 99, + 111, 109, 112, 111, 115, 101, + 73, 110, 100, 101, 120, 32, + 114, 101, 116, 117, 114, 110, + 32, 118, 97, 108, 117, 101, + 62, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 0, 0, 4, 0, 140, 2, + 0, 0, 1, 0, 60, 0, + 32, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 4, 0, 4, 0, 160, 2, + 0, 0, 1, 0, 40, 0, + 36, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 8, 0, 4, 0, 180, 2, + 0, 0, 1, 0, 20, 0, + 40, 0, 0, 0, 46, 0, + 62, 17, 117, 0, 0, 0, + 1, 0, 105, 110, 100, 101, + 120, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 1, 0, + 0, 0, 4, 0, 160, 1, + 0, 0, 1, 0, 40, 1, + 4, 0, 0, 0, 42, 0, + 62, 17, 117, 0, 0, 0, + 0, 0, 116, 101, 109, 112, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 1, 0, 0, 0, 4, 0, + 232, 1, 0, 0, 1, 0, + 224, 0, 20, 0, 0, 0, + 42, 0, 62, 17, 0, 16, + 0, 0, 8, 0, 105, 100, + 120, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 48, 2, 0, 0, + 1, 0, 152, 0, 32, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 84, 2, 0, 0, + 1, 0, 116, 0, 36, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 8, 0, + 4, 0, 120, 2, 0, 0, + 1, 0, 80, 0, 40, 0, + 0, 0, 2, 0, 78, 17, + 174, 0, 77, 17, 124, 0, + 0, 0, 184, 11, 0, 0, + 1, 16, 0, 0, 7, 0, + 9, 5, 13, 32, 6, 2, + 3, 131, 108, 7, 3, 13, + 34, 6, 4, 3, 20, 7, + 0, 13, 29, 6, 8, 3, + 128, 164, 13, 27, 6, 8, + 3, 44, 9, 9, 13, 58, + 6, 2, 3, 64, 13, 38, + 6, 2, 3, 32, 9, 5, + 13, 5, 6, 2, 3, 44, + 13, 17, 11, 68, 4, 40, + 8, 0, 9, 20, 13, 31, + 1, 131, 200, 6, 31, 3, + 0, 9, 9, 13, 33, 6, + 6, 3, 20, 6, 2, 3, + 36, 7, 1, 6, 3, 3, + 36, 7, 0, 6, 4, 3, + 28, 7, 2, 6, 5, 3, + 36, 7, 0, 9, 15, 13, + 28, 6, 6, 3, 28, 9, + 9, 13, 26, 6, 8, 3, + 44, 9, 5, 13, 27, 3, + 52, 9, 31, 13, 57, 11, + 44, 9, 19, 13, 37, 6, + 2, 3, 32, 9, 5, 13, + 5, 6, 2, 3, 44, 13, + 17, 11, 68, 4, 40, 0, + 0, 0, 70, 0, 62, 17, + 10, 16, 0, 0, 136, 0, + 60, 82, 101, 97, 100, 83, + 111, 117, 114, 99, 101, 86, + 97, 108, 117, 101, 32, 114, + 101, 116, 117, 114, 110, 32, + 118, 97, 108, 117, 101, 62, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 80, 5, 0, 0, + 1, 0, 40, 0, 64, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 100, 5, 0, 0, + 1, 0, 20, 0, 68, 0, + 0, 0, 46, 0, 62, 17, + 0, 16, 0, 0, 9, 0, + 105, 110, 100, 101, 120, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 200, 3, 0, 0, + 1, 0, 92, 0, 16, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 200, 3, 0, 0, + 1, 0, 176, 1, 20, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 8, 0, + 4, 0, 200, 3, 0, 0, + 1, 0, 176, 1, 24, 0, + 0, 0, 46, 0, 62, 17, + 5, 16, 0, 0, 8, 0, + 118, 97, 108, 117, 101, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 220, 3, 0, 0, + 1, 0, 156, 1, 68, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 172, 4, 0, 0, + 1, 0, 204, 0, 64, 0, + 0, 0, 50, 0, 62, 17, + 117, 0, 0, 0, 0, 0, + 105, 110, 100, 101, 120, 82, + 101, 97, 108, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 1, 0, 0, 0, 4, 0, + 128, 4, 0, 0, 1, 0, + 140, 0, 8, 0, 0, 0, + 54, 0, 62, 17, 117, 0, + 0, 0, 0, 0, 105, 110, + 100, 101, 120, 73, 109, 97, + 103, 105, 110, 97, 114, 121, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 12, 5, 0, 0, + 1, 0, 48, 0, 8, 0, + 0, 0, 2, 0, 78, 17, + 174, 0, 77, 17, 124, 0, + 0, 0, 160, 14, 0, 0, + 1, 16, 0, 0, 7, 0, + 9, 5, 13, 32, 6, 2, + 3, 133, 52, 7, 3, 13, + 34, 6, 4, 3, 20, 7, + 0, 13, 29, 6, 8, 3, + 128, 164, 13, 27, 6, 8, + 3, 44, 9, 9, 13, 58, + 6, 2, 3, 64, 13, 38, + 6, 2, 3, 32, 9, 5, + 13, 5, 6, 2, 3, 44, + 13, 17, 11, 68, 4, 40, + 8, 0, 9, 20, 13, 31, + 1, 133, 144, 6, 31, 3, + 0, 9, 9, 13, 33, 6, + 6, 3, 20, 6, 2, 3, + 36, 7, 1, 6, 3, 3, + 36, 7, 0, 6, 4, 3, + 28, 7, 2, 6, 5, 3, + 36, 7, 0, 9, 15, 13, + 28, 6, 6, 3, 28, 9, + 9, 13, 26, 6, 8, 3, + 44, 9, 5, 13, 27, 3, + 52, 9, 31, 13, 57, 11, + 44, 9, 19, 13, 37, 6, + 2, 3, 32, 9, 5, 13, + 5, 6, 2, 3, 44, 13, + 17, 11, 68, 4, 40, 0, + 0, 0, 70, 0, 62, 17, + 10, 16, 0, 0, 136, 0, + 60, 82, 101, 97, 100, 83, + 111, 117, 114, 99, 101, 86, + 97, 108, 117, 101, 32, 114, + 101, 116, 117, 114, 110, 32, + 118, 97, 108, 117, 101, 62, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 144, 5, 0, 0, + 1, 0, 136, 1, 64, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 144, 5, 0, 0, + 1, 0, 156, 1, 68, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 24, 7, 0, 0, + 1, 0, 40, 0, 16, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 44, 7, 0, 0, + 1, 0, 20, 0, 20, 0, + 0, 0, 46, 0, 62, 17, + 0, 16, 0, 0, 9, 0, + 105, 110, 100, 101, 120, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 144, 5, 0, 0, + 1, 0, 176, 1, 48, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 144, 5, 0, 0, + 1, 0, 176, 1, 52, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 8, 0, + 4, 0, 144, 5, 0, 0, + 1, 0, 176, 1, 56, 0, + 0, 0, 46, 0, 62, 17, + 5, 16, 0, 0, 8, 0, + 118, 97, 108, 117, 101, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 144, 5, 0, 0, + 1, 0, 20, 0, 68, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 164, 5, 0, 0, + 1, 0, 156, 1, 20, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 116, 6, 0, 0, + 1, 0, 204, 0, 16, 0, + 0, 0, 50, 0, 62, 17, + 117, 0, 0, 0, 0, 0, + 105, 110, 100, 101, 120, 82, + 101, 97, 108, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 22, 0, 80, 17, 0, 0, + 1, 0, 0, 0, 4, 0, + 72, 6, 0, 0, 1, 0, + 140, 0, 8, 0, 0, 0, + 54, 0, 62, 17, 117, 0, + 0, 0, 0, 0, 105, 110, + 100, 101, 120, 73, 109, 97, + 103, 105, 110, 97, 114, 121, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 212, 6, 0, 0, + 1, 0, 48, 0, 8, 0, + 0, 0, 2, 0, 78, 17, + 66, 0, 77, 17, 124, 0, + 0, 0, 16, 16, 0, 0, + 2, 16, 0, 0, 7, 0, + 9, 5, 13, 62, 6, 2, + 3, 136, 32, 13, 59, 6, + 2, 3, 36, 13, 26, 6, + 2, 12, 40, 32, 8, 0, + 9, 34, 13, 57, 1, 136, + 124, 11, 80, 9, 24, 13, + 58, 6, 2, 3, 36, 9, + 5, 13, 26, 6, 2, 12, + 40, 32, 70, 0, 62, 17, + 12, 16, 0, 0, 136, 0, + 60, 67, 111, 109, 112, 117, + 116, 101, 68, 101, 115, 116, + 73, 110, 100, 101, 120, 32, + 114, 101, 116, 117, 114, 110, + 32, 118, 97, 108, 117, 101, + 62, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 0, 0, + 4, 0, 212, 8, 0, 0, + 1, 0, 40, 0, 32, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 5, 0, 4, 0, + 4, 0, 232, 8, 0, 0, + 1, 0, 20, 0, 36, 0, + 0, 0, 46, 0, 62, 17, + 117, 0, 0, 0, 1, 0, + 105, 110, 100, 101, 120, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 80, 17, + 0, 0, 1, 0, 0, 0, + 4, 0, 124, 8, 0, 0, + 1, 0, 128, 0, 4, 0, + 0, 0, 54, 0, 62, 17, + 4, 16, 0, 0, 8, 0, + 100, 102, 116, 79, 117, 116, + 112, 117, 116, 73, 110, 100, + 101, 120, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 0, 0, 4, 0, 160, 8, + 0, 0, 1, 0, 92, 0, + 32, 0, 0, 0, 22, 0, + 80, 17, 0, 0, 5, 0, + 4, 0, 4, 0, 192, 8, + 0, 0, 1, 0, 60, 0, + 36, 0, 0, 0, 2, 0, + 78, 17, 2, 0, 6, 0, + 244, 0, 0, 0, 24, 0, + 0, 0, 1, 0, 0, 0, + 16, 1, 164, 236, 135, 157, + 135, 121, 170, 157, 96, 113, + 137, 51, 222, 23, 161, 191, + 0, 0, 242, 0, 0, 0, + 168, 9, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, + 132, 10, 0, 0, 0, 0, + 0, 0, 204, 0, 0, 0, + 156, 9, 0, 0, 92, 0, + 0, 0, 87, 0, 0, 128, + 92, 0, 0, 0, 87, 0, + 0, 0, 112, 0, 0, 0, + 61, 0, 0, 128, 112, 0, + 0, 0, 61, 0, 0, 0, + 140, 0, 0, 0, 62, 0, + 0, 128, 140, 0, 0, 0, + 62, 0, 0, 0, 172, 0, + 0, 0, 62, 0, 0, 128, + 172, 0, 0, 0, 62, 0, + 0, 0, 184, 0, 0, 0, + 64, 0, 0, 128, 184, 0, + 0, 0, 64, 0, 0, 0, + 204, 0, 0, 0, 64, 0, + 0, 128, 204, 0, 0, 0, + 64, 0, 0, 0, 240, 0, + 0, 0, 65, 0, 0, 128, + 240, 0, 0, 0, 65, 0, + 0, 0, 8, 1, 0, 0, + 65, 0, 0, 128, 8, 1, + 0, 0, 65, 0, 0, 0, + 36, 1, 0, 0, 66, 0, + 0, 128, 36, 1, 0, 0, + 66, 0, 0, 0, 56, 1, + 0, 0, 66, 0, 0, 128, + 56, 1, 0, 0, 66, 0, + 0, 0, 76, 1, 0, 0, + 66, 0, 0, 128, 76, 1, + 0, 0, 66, 0, 0, 0, + 108, 1, 0, 0, 66, 0, + 0, 128, 108, 1, 0, 0, + 66, 0, 0, 0, 136, 1, + 0, 0, 71, 0, 0, 128, + 136, 1, 0, 0, 71, 0, + 0, 0, 140, 1, 0, 0, + 71, 0, 0, 128, 140, 1, + 0, 0, 71, 0, 0, 0, + 160, 1, 0, 0, 71, 0, + 0, 128, 160, 1, 0, 0, + 71, 0, 0, 0, 200, 1, + 0, 0, 71, 0, 0, 128, + 200, 1, 0, 0, 71, 0, + 0, 0, 232, 1, 0, 0, + 71, 0, 0, 128, 232, 1, + 0, 0, 71, 0, 0, 0, + 16, 2, 0, 0, 71, 0, + 0, 128, 16, 2, 0, 0, + 71, 0, 0, 0, 48, 2, + 0, 0, 71, 0, 0, 128, + 48, 2, 0, 0, 71, 0, + 0, 0, 84, 2, 0, 0, + 71, 0, 0, 128, 84, 2, + 0, 0, 71, 0, 0, 0, + 120, 2, 0, 0, 71, 0, + 0, 128, 120, 2, 0, 0, + 71, 0, 0, 0, 140, 2, + 0, 0, 71, 0, 0, 128, + 140, 2, 0, 0, 71, 0, + 0, 0, 160, 2, 0, 0, + 71, 0, 0, 128, 160, 2, + 0, 0, 71, 0, 0, 0, + 180, 2, 0, 0, 71, 0, + 0, 128, 180, 2, 0, 0, + 71, 0, 0, 0, 200, 2, + 0, 0, 72, 0, 0, 128, + 200, 2, 0, 0, 72, 0, + 0, 0, 232, 2, 0, 0, + 72, 0, 0, 128, 232, 2, + 0, 0, 72, 0, 0, 0, + 8, 3, 0, 0, 72, 0, + 0, 128, 8, 3, 0, 0, + 72, 0, 0, 0, 40, 3, + 0, 0, 72, 0, 0, 128, + 40, 3, 0, 0, 72, 0, + 0, 0, 68, 3, 0, 0, + 73, 0, 0, 128, 68, 3, + 0, 0, 73, 0, 0, 0, + 96, 3, 0, 0, 76, 0, + 0, 128, 96, 3, 0, 0, + 76, 0, 0, 0, 116, 3, + 0, 0, 76, 0, 0, 128, + 116, 3, 0, 0, 76, 0, + 0, 0, 136, 3, 0, 0, + 77, 0, 0, 128, 136, 3, + 0, 0, 77, 0, 0, 0, + 156, 3, 0, 0, 77, 0, + 0, 128, 156, 3, 0, 0, + 77, 0, 0, 0, 176, 3, + 0, 0, 80, 0, 0, 128, + 176, 3, 0, 0, 80, 0, + 0, 0, 180, 3, 0, 0, + 80, 0, 0, 128, 180, 3, + 0, 0, 80, 0, 0, 0, + 200, 3, 0, 0, 80, 0, + 0, 128, 200, 3, 0, 0, + 80, 0, 0, 0, 220, 3, + 0, 0, 80, 0, 0, 128, + 220, 3, 0, 0, 80, 0, + 0, 0, 0, 4, 0, 0, + 80, 0, 0, 128, 0, 4, + 0, 0, 80, 0, 0, 0, + 36, 4, 0, 0, 80, 0, + 0, 128, 36, 4, 0, 0, + 80, 0, 0, 0, 64, 4, + 0, 0, 80, 0, 0, 128, + 64, 4, 0, 0, 80, 0, + 0, 0, 100, 4, 0, 0, + 80, 0, 0, 128, 100, 4, + 0, 0, 80, 0, 0, 0, + 128, 4, 0, 0, 80, 0, + 0, 128, 128, 4, 0, 0, + 80, 0, 0, 0, 172, 4, + 0, 0, 80, 0, 0, 128, + 172, 4, 0, 0, 80, 0, + 0, 0, 192, 4, 0, 0, + 80, 0, 0, 128, 192, 4, + 0, 0, 80, 0, 0, 0, + 224, 4, 0, 0, 80, 0, + 0, 128, 224, 4, 0, 0, + 80, 0, 0, 0, 236, 4, + 0, 0, 80, 0, 0, 128, + 236, 4, 0, 0, 80, 0, + 0, 0, 12, 5, 0, 0, + 80, 0, 0, 128, 12, 5, + 0, 0, 80, 0, 0, 0, + 56, 5, 0, 0, 80, 0, + 0, 128, 56, 5, 0, 0, + 80, 0, 0, 0, 60, 5, + 0, 0, 80, 0, 0, 128, + 60, 5, 0, 0, 80, 0, + 0, 0, 80, 5, 0, 0, + 80, 0, 0, 128, 80, 5, + 0, 0, 80, 0, 0, 0, + 100, 5, 0, 0, 80, 0, + 0, 128, 100, 5, 0, 0, + 80, 0, 0, 0, 120, 5, + 0, 0, 81, 0, 0, 128, + 120, 5, 0, 0, 81, 0, + 0, 0, 124, 5, 0, 0, + 81, 0, 0, 128, 124, 5, + 0, 0, 81, 0, 0, 0, + 144, 5, 0, 0, 81, 0, + 0, 128, 144, 5, 0, 0, + 81, 0, 0, 0, 164, 5, + 0, 0, 81, 0, 0, 128, + 164, 5, 0, 0, 81, 0, + 0, 0, 200, 5, 0, 0, + 81, 0, 0, 128, 200, 5, + 0, 0, 81, 0, 0, 0, + 236, 5, 0, 0, 81, 0, + 0, 128, 236, 5, 0, 0, + 81, 0, 0, 0, 8, 6, + 0, 0, 81, 0, 0, 128, + 8, 6, 0, 0, 81, 0, + 0, 0, 44, 6, 0, 0, + 81, 0, 0, 128, 44, 6, + 0, 0, 81, 0, 0, 0, + 72, 6, 0, 0, 81, 0, + 0, 128, 72, 6, 0, 0, + 81, 0, 0, 0, 116, 6, + 0, 0, 81, 0, 0, 128, + 116, 6, 0, 0, 81, 0, + 0, 0, 136, 6, 0, 0, + 81, 0, 0, 128, 136, 6, + 0, 0, 81, 0, 0, 0, + 168, 6, 0, 0, 81, 0, + 0, 128, 168, 6, 0, 0, + 81, 0, 0, 0, 180, 6, + 0, 0, 81, 0, 0, 128, + 180, 6, 0, 0, 81, 0, + 0, 0, 212, 6, 0, 0, + 81, 0, 0, 128, 212, 6, + 0, 0, 81, 0, 0, 0, + 0, 7, 0, 0, 81, 0, + 0, 128, 0, 7, 0, 0, + 81, 0, 0, 0, 4, 7, + 0, 0, 81, 0, 0, 128, + 4, 7, 0, 0, 81, 0, + 0, 0, 24, 7, 0, 0, + 81, 0, 0, 128, 24, 7, + 0, 0, 81, 0, 0, 0, + 44, 7, 0, 0, 81, 0, + 0, 128, 44, 7, 0, 0, + 81, 0, 0, 0, 64, 7, + 0, 0, 85, 0, 0, 128, + 64, 7, 0, 0, 85, 0, + 0, 0, 96, 7, 0, 0, + 88, 0, 0, 128, 96, 7, + 0, 0, 88, 0, 0, 0, + 116, 7, 0, 0, 88, 0, + 0, 128, 116, 7, 0, 0, + 88, 0, 0, 0, 148, 7, + 0, 0, 89, 0, 0, 128, + 148, 7, 0, 0, 89, 0, + 0, 0, 184, 7, 0, 0, + 90, 0, 0, 128, 184, 7, + 0, 0, 90, 0, 0, 0, + 212, 7, 0, 0, 90, 0, + 0, 128, 212, 7, 0, 0, + 90, 0, 0, 0, 232, 7, + 0, 0, 90, 0, 0, 128, + 232, 7, 0, 0, 90, 0, + 0, 0, 4, 8, 0, 0, + 90, 0, 0, 128, 4, 8, + 0, 0, 90, 0, 0, 0, + 24, 8, 0, 0, 90, 0, + 0, 128, 24, 8, 0, 0, + 90, 0, 0, 0, 52, 8, + 0, 0, 91, 0, 0, 128, + 52, 8, 0, 0, 91, 0, + 0, 0, 76, 8, 0, 0, + 91, 0, 0, 128, 76, 8, + 0, 0, 91, 0, 0, 0, + 100, 8, 0, 0, 93, 0, + 0, 128, 100, 8, 0, 0, + 93, 0, 0, 0, 104, 8, + 0, 0, 93, 0, 0, 128, + 104, 8, 0, 0, 93, 0, + 0, 0, 124, 8, 0, 0, + 93, 0, 0, 128, 124, 8, + 0, 0, 93, 0, 0, 0, + 160, 8, 0, 0, 93, 0, + 0, 128, 160, 8, 0, 0, + 93, 0, 0, 0, 192, 8, + 0, 0, 93, 0, 0, 128, + 192, 8, 0, 0, 93, 0, + 0, 0, 212, 8, 0, 0, + 93, 0, 0, 128, 212, 8, + 0, 0, 93, 0, 0, 0, + 232, 8, 0, 0, 93, 0, + 0, 128, 232, 8, 0, 0, + 93, 0, 0, 0, 252, 8, + 0, 0, 94, 0, 0, 128, + 252, 8, 0, 0, 94, 0, + 0, 0, 24, 9, 0, 0, + 94, 0, 0, 128, 24, 9, + 0, 0, 94, 0, 0, 0, + 52, 9, 0, 0, 94, 0, + 0, 128, 52, 9, 0, 0, + 94, 0, 0, 0, 76, 9, + 0, 0, 94, 0, 0, 128, + 76, 9, 0, 0, 94, 0, + 0, 0, 104, 9, 0, 0, + 94, 0, 0, 128, 104, 9, + 0, 0, 94, 0, 0, 0, + 132, 9, 0, 0, 94, 0, + 0, 128, 132, 9, 0, 0, + 94, 0, 0, 0, 164, 9, + 0, 0, 94, 0, 0, 128, + 164, 9, 0, 0, 94, 0, + 0, 0, 200, 9, 0, 0, + 95, 0, 0, 128, 200, 9, + 0, 0, 95, 0, 0, 0, + 228, 9, 0, 0, 95, 0, + 0, 128, 228, 9, 0, 0, + 95, 0, 0, 0, 0, 10, + 0, 0, 95, 0, 0, 128, + 0, 10, 0, 0, 95, 0, + 0, 0, 28, 10, 0, 0, + 95, 0, 0, 128, 28, 10, + 0, 0, 95, 0, 0, 0, + 56, 10, 0, 0, 95, 0, + 0, 128, 56, 10, 0, 0, + 95, 0, 0, 0, 88, 10, + 0, 0, 95, 0, 0, 128, + 88, 10, 0, 0, 95, 0, + 0, 0, 124, 10, 0, 0, + 96, 0, 0, 128, 124, 10, + 0, 0, 96, 0, 0, 0, + 128, 10, 0, 0, 97, 0, + 0, 128, 128, 10, 0, 0, + 97, 0, 0, 0, 28, 0, + 39, 0, 28, 0, 39, 0, + 5, 0, 37, 0, 18, 0, + 36, 0, 5, 0, 29, 0, + 9, 0, 28, 0, 5, 0, + 29, 0, 5, 0, 29, 0, + 9, 0, 48, 0, 35, 0, + 47, 0, 9, 0, 48, 0, + 35, 0, 47, 0, 9, 0, + 35, 0, 18, 0, 34, 0, + 9, 0, 35, 0, 18, 0, + 34, 0, 9, 0, 45, 0, + 28, 0, 43, 0, 9, 0, + 45, 0, 28, 0, 43, 0, + 9, 0, 45, 0, 28, 0, + 43, 0, 9, 0, 45, 0, + 22, 0, 44, 0, 9, 0, + 42, 0, 21, 0, 41, 0, + 9, 0, 42, 0, 21, 0, + 41, 0, 9, 0, 42, 0, + 21, 0, 41, 0, 9, 0, + 42, 0, 21, 0, 41, 0, + 9, 0, 42, 0, 21, 0, + 41, 0, 9, 0, 42, 0, + 21, 0, 41, 0, 9, 0, + 42, 0, 21, 0, 41, 0, + 9, 0, 42, 0, 21, 0, + 41, 0, 9, 0, 42, 0, + 21, 0, 41, 0, 9, 0, + 42, 0, 21, 0, 41, 0, + 9, 0, 42, 0, 21, 0, + 41, 0, 9, 0, 42, 0, + 15, 0, 41, 0, 9, 0, + 84, 0, 36, 0, 56, 0, + 9, 0, 84, 0, 35, 0, + 65, 0, 9, 0, 84, 0, + 70, 0, 82, 0, 9, 0, + 84, 0, 35, 0, 83, 0, + 9, 0, 79, 0, 35, 0, + 78, 0, 9, 0, 74, 0, + 15, 0, 73, 0, 9, 0, + 74, 0, 15, 0, 73, 0, + 9, 0, 73, 0, 15, 0, + 72, 0, 9, 0, 73, 0, + 15, 0, 72, 0, 9, 0, + 62, 0, 33, 0, 61, 0, + 9, 0, 62, 0, 33, 0, + 61, 0, 9, 0, 62, 0, + 33, 0, 61, 0, 9, 0, + 62, 0, 33, 0, 61, 0, + 9, 0, 62, 0, 33, 0, + 61, 0, 9, 0, 62, 0, + 33, 0, 61, 0, 9, 0, + 62, 0, 33, 0, 61, 0, + 9, 0, 62, 0, 33, 0, + 61, 0, 9, 0, 62, 0, + 33, 0, 61, 0, 9, 0, + 62, 0, 33, 0, 61, 0, + 9, 0, 62, 0, 33, 0, + 61, 0, 9, 0, 62, 0, + 33, 0, 61, 0, 9, 0, + 62, 0, 33, 0, 61, 0, + 9, 0, 62, 0, 33, 0, + 61, 0, 9, 0, 62, 0, + 33, 0, 61, 0, 9, 0, + 62, 0, 33, 0, 61, 0, + 9, 0, 62, 0, 33, 0, + 61, 0, 9, 0, 62, 0, + 16, 0, 61, 0, 9, 0, + 60, 0, 32, 0, 59, 0, + 9, 0, 60, 0, 32, 0, + 59, 0, 9, 0, 60, 0, + 32, 0, 59, 0, 9, 0, + 60, 0, 32, 0, 59, 0, + 9, 0, 60, 0, 32, 0, + 59, 0, 9, 0, 60, 0, + 32, 0, 59, 0, 9, 0, + 60, 0, 32, 0, 59, 0, + 9, 0, 60, 0, 32, 0, + 59, 0, 9, 0, 60, 0, + 32, 0, 59, 0, 9, 0, + 60, 0, 32, 0, 59, 0, + 9, 0, 60, 0, 32, 0, + 59, 0, 9, 0, 60, 0, + 32, 0, 59, 0, 9, 0, + 60, 0, 32, 0, 59, 0, + 9, 0, 60, 0, 32, 0, + 59, 0, 9, 0, 60, 0, + 32, 0, 59, 0, 9, 0, + 60, 0, 32, 0, 59, 0, + 9, 0, 60, 0, 32, 0, + 59, 0, 9, 0, 60, 0, + 16, 0, 59, 0, 9, 0, + 27, 0, 18, 0, 26, 0, + 9, 0, 40, 0, 26, 0, + 39, 0, 9, 0, 40, 0, + 26, 0, 39, 0, 9, 0, + 63, 0, 41, 0, 62, 0, + 9, 0, 68, 0, 23, 0, + 45, 0, 9, 0, 68, 0, + 49, 0, 56, 0, 9, 0, + 68, 0, 23, 0, 56, 0, + 9, 0, 68, 0, 60, 0, + 67, 0, 9, 0, 68, 0, + 23, 0, 67, 0, 9, 0, + 50, 0, 27, 0, 36, 0, + 9, 0, 50, 0, 39, 0, + 48, 0, 9, 0, 52, 0, + 29, 0, 51, 0, 9, 0, + 52, 0, 29, 0, 51, 0, + 9, 0, 52, 0, 29, 0, + 51, 0, 9, 0, 52, 0, + 29, 0, 51, 0, 9, 0, + 52, 0, 29, 0, 51, 0, + 9, 0, 52, 0, 29, 0, + 51, 0, 9, 0, 52, 0, + 15, 0, 51, 0, 9, 0, + 106, 0, 59, 0, 79, 0, + 9, 0, 106, 0, 83, 0, + 103, 0, 9, 0, 106, 0, + 59, 0, 103, 0, 9, 0, + 106, 0, 59, 0, 103, 0, + 9, 0, 106, 0, 39, 0, + 104, 0, 9, 0, 106, 0, + 30, 0, 105, 0, 9, 0, + 106, 0, 9, 0, 105, 0, + 9, 0, 106, 0, 59, 0, + 79, 0, 9, 0, 106, 0, + 83, 0, 103, 0, 9, 0, + 106, 0, 59, 0, 103, 0, + 9, 0, 106, 0, 39, 0, + 104, 0, 9, 0, 106, 0, + 30, 0, 105, 0, 9, 0, + 106, 0, 9, 0, 105, 0, + 5, 0, 5, 0, 5, 0, + 5, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 246, 0, + 0, 0, 52, 0, 0, 0, + 0, 0, 0, 0, 0, 16, + 0, 0, 0, 0, 0, 0, + 48, 0, 0, 0, 1, 16, + 0, 0, 0, 0, 0, 0, + 28, 0, 0, 0, 1, 16, + 0, 0, 0, 0, 0, 0, + 28, 0, 0, 0, 2, 16, + 0, 0, 0, 0, 0, 0, + 20, 0, 0, 0, 52, 0, + 0, 0, 0, 0, 0, 0, + 20, 0, 0, 0, 44, 0, + 0, 0, 68, 0, 0, 0, + 100, 0, 0, 0, 136, 0, + 0, 0, 172, 0, 0, 0, + 204, 0, 0, 0, 236, 0, + 0, 0, 16, 1, 0, 0, + 48, 1, 0, 0, 84, 1, + 0, 0, 112, 1, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 11, 202, 49, 1, 56, 0, + 0, 0, 0, 16, 0, 0, + 22, 16, 0, 0, 64, 1, + 0, 0, 10, 0, 255, 255, + 4, 0, 0, 0, 255, 255, + 3, 0, 0, 0, 0, 0, + 88, 0, 0, 0, 88, 0, + 0, 0, 8, 0, 0, 0, + 96, 0, 0, 0, 0, 0, + 0, 0, 18, 0, 27, 21, + 117, 0, 0, 0, 3, 0, + 0, 0, 12, 0, 117, 105, + 110, 116, 51, 0, 10, 0, + 1, 18, 1, 0, 0, 0, + 0, 16, 0, 0, 10, 0, + 24, 21, 3, 0, 0, 0, + 1, 0, 1, 0, 14, 0, + 8, 16, 2, 16, 0, 0, + 23, 0, 1, 0, 1, 16, + 0, 0, 18, 0, 27, 21, + 117, 0, 0, 0, 2, 0, + 0, 0, 8, 0, 117, 105, + 110, 116, 50, 0, 22, 0, + 27, 21, 64, 0, 0, 0, + 2, 0, 0, 0, 8, 0, + 102, 108, 111, 97, 116, 50, + 0, 243, 242, 241, 10, 0, + 24, 21, 64, 0, 0, 0, + 1, 0, 1, 0, 10, 0, + 1, 18, 1, 0, 0, 0, + 117, 0, 0, 0, 10, 0, + 24, 21, 0, 16, 0, 0, + 1, 0, 1, 0, 14, 0, + 8, 16, 8, 16, 0, 0, + 23, 0, 1, 0, 7, 16, + 0, 0, 10, 0, 24, 21, + 5, 16, 0, 0, 1, 0, + 1, 0, 14, 0, 8, 16, + 10, 16, 0, 0, 23, 0, + 1, 0, 1, 16, 0, 0, + 10, 0, 24, 21, 4, 16, + 0, 0, 1, 0, 1, 0, + 14, 0, 8, 16, 12, 16, + 0, 0, 23, 0, 1, 0, + 7, 16, 0, 0, 14, 0, + 23, 21, 64, 0, 0, 0, + 27, 2, 0, 0, 0, 0, + 242, 241, 10, 0, 24, 21, + 14, 16, 0, 0, 1, 0, + 0, 2, 10, 0, 24, 21, + 117, 0, 0, 0, 1, 0, + 1, 0, 10, 0, 24, 21, + 16, 16, 0, 0, 1, 0, + 0, 2, 18, 0, 27, 21, + 117, 0, 0, 0, 4, 0, + 0, 0, 16, 0, 117, 105, + 110, 116, 52, 0, 10, 0, + 24, 21, 18, 16, 0, 0, + 1, 0, 1, 0, 10, 0, + 24, 21, 19, 16, 0, 0, + 1, 0, 0, 2, 10, 0, + 24, 21, 6, 16, 0, 0, + 1, 0, 0, 2, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 219, 114, 0, 0, + 17, 39, 1, 0, 4, 146, + 3, 0, 0, 16, 0, 0, + 0, 0, 0, 0, 255, 255, + 255, 255, 4, 0, 0, 0, + 255, 255, 3, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 255, 255, + 255, 255, 0, 0, 0, 0, + 255, 255, 255, 255, 18, 0, + 27, 21, 117, 0, 0, 0, + 3, 0, 0, 0, 12, 0, + 117, 105, 110, 116, 51, 0, + 10, 0, 1, 18, 1, 0, + 0, 0, 0, 16, 0, 0, + 10, 0, 24, 21, 3, 0, + 0, 0, 1, 0, 1, 0, + 14, 0, 8, 16, 2, 16, + 0, 0, 23, 0, 1, 0, + 1, 16, 0, 0, 18, 0, + 27, 21, 117, 0, 0, 0, + 2, 0, 0, 0, 8, 0, + 117, 105, 110, 116, 50, 0, + 22, 0, 27, 21, 64, 0, + 0, 0, 2, 0, 0, 0, + 8, 0, 102, 108, 111, 97, + 116, 50, 0, 243, 242, 241, + 10, 0, 24, 21, 64, 0, + 0, 0, 1, 0, 1, 0, + 10, 0, 1, 18, 1, 0, + 0, 0, 117, 0, 0, 0, + 10, 0, 24, 21, 0, 16, + 0, 0, 1, 0, 1, 0, + 14, 0, 8, 16, 8, 16, + 0, 0, 23, 0, 1, 0, + 7, 16, 0, 0, 10, 0, + 24, 21, 5, 16, 0, 0, + 1, 0, 1, 0, 14, 0, + 8, 16, 10, 16, 0, 0, + 23, 0, 1, 0, 1, 16, + 0, 0, 10, 0, 24, 21, + 4, 16, 0, 0, 1, 0, + 1, 0, 14, 0, 8, 16, + 12, 16, 0, 0, 23, 0, + 1, 0, 7, 16, 0, 0, + 14, 0, 23, 21, 64, 0, + 0, 0, 27, 2, 0, 0, + 0, 0, 242, 241, 10, 0, + 24, 21, 14, 16, 0, 0, + 1, 0, 0, 2, 10, 0, + 24, 21, 117, 0, 0, 0, + 1, 0, 1, 0, 10, 0, + 24, 21, 16, 16, 0, 0, + 1, 0, 0, 2, 18, 0, + 27, 21, 117, 0, 0, 0, + 4, 0, 0, 0, 16, 0, + 117, 105, 110, 116, 52, 0, + 10, 0, 24, 21, 18, 16, + 0, 0, 1, 0, 1, 0, + 10, 0, 24, 21, 19, 16, + 0, 0, 1, 0, 0, 2, + 10, 0, 24, 21, 6, 16, + 0, 0, 1, 0, 0, 2, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 68, 51, + 68, 83, 72, 68, 82, 0, + 132, 10, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 32, 0, + 0, 96, 12, 0, 0, 0, + 8, 0, 0, 0, 20, 0, + 0, 0, 0, 0, 0, 0, + 26, 0, 1, 22, 0, 0, + 0, 0, 9, 16, 0, 0, + 68, 101, 99, 111, 109, 112, + 111, 115, 101, 73, 110, 100, + 101, 120, 0, 241, 26, 0, + 1, 22, 0, 0, 0, 0, + 11, 16, 0, 0, 82, 101, + 97, 100, 83, 111, 117, 114, + 99, 101, 86, 97, 108, 117, + 101, 0, 30, 0, 1, 22, + 0, 0, 0, 0, 13, 16, + 0, 0, 67, 111, 109, 112, + 117, 116, 101, 68, 101, 115, + 116, 73, 110, 100, 101, 120, + 0, 243, 242, 241, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 255, 255, 255, 255, 26, 9, + 47, 241, 104, 0, 0, 0, + 56, 2, 0, 0, 137, 0, + 0, 0, 1, 0, 0, 0, + 237, 0, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 0, + 1, 0, 0, 0, 101, 0, + 0, 0, 1, 0, 0, 0, + 85, 1, 0, 0, 1, 0, + 0, 0, 49, 1, 0, 0, + 1, 0, 0, 0, 21, 0, + 0, 0, 1, 0, 0, 0, + 113, 1, 0, 0, 1, 0, + 0, 0, 205, 0, 0, 0, + 1, 0, 0, 0, 69, 0, + 0, 0, 1, 0, 0, 0, + 45, 0, 0, 0, 1, 0, + 0, 0, 17, 1, 0, 0, + 1, 0, 0, 0, 173, 0, + 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 16, 16, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 64, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 4, 0, 0, + 0, 1, 0, 0, 0, 0, + 0, 2, 0, 0, 0, 0, + 0, 0, 128, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 4, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 128, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 12, 0, + 0, 0, 24, 0, 0, 0, + 36, 0, 0, 0, 48, 0, + 0, 0, 60, 0, 0, 0, + 72, 0, 0, 0, 84, 0, + 0, 0, 96, 0, 0, 0, + 108, 0, 0, 0, 120, 0, + 0, 0, 132, 0, 0, 0, + 144, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 18, 0, + 37, 17, 0, 0, 0, 0, + 124, 0, 0, 0, 1, 0, + 68, 70, 84, 0, 0, 0, + 22, 0, 81, 17, 15, 16, + 0, 0, 30, 0, 255, 255, + 255, 255, 255, 255, 255, 255, + 0, 0, 115, 114, 99, 0, + 22, 0, 81, 17, 15, 16, + 0, 0, 30, 0, 255, 255, + 255, 255, 255, 255, 255, 255, + 1, 0, 100, 115, 116, 0, + 30, 0, 81, 17, 17, 16, + 0, 0, 8, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 255, 255, 83, 116, 97, 114, + 116, 73, 110, 100, 101, 120, + 0, 0, 34, 0, 81, 17, + 17, 16, 0, 0, 8, 0, + 0, 0, 4, 0, 255, 255, + 255, 255, 255, 255, 69, 108, + 101, 109, 101, 110, 116, 67, + 111, 117, 110, 116, 0, 0, + 0, 0, 34, 0, 81, 17, + 17, 16, 0, 0, 8, 0, + 0, 0, 8, 0, 255, 255, + 255, 255, 255, 255, 68, 70, + 84, 73, 116, 101, 114, 97, + 116, 105, 111, 110, 0, 0, + 0, 0, 30, 0, 81, 17, + 17, 16, 0, 0, 8, 0, + 0, 0, 12, 0, 255, 255, + 255, 255, 255, 255, 73, 115, + 73, 110, 118, 101, 114, 115, + 101, 0, 0, 0, 30, 0, + 81, 17, 20, 16, 0, 0, + 8, 0, 0, 0, 16, 0, + 255, 255, 255, 255, 255, 255, + 73, 110, 112, 117, 116, 83, + 105, 122, 101, 115, 0, 0, + 34, 0, 81, 17, 20, 16, + 0, 0, 8, 0, 0, 0, + 32, 0, 255, 255, 255, 255, + 255, 255, 73, 110, 112, 117, + 116, 83, 116, 114, 105, 100, + 101, 115, 0, 0, 0, 0, + 30, 0, 81, 17, 20, 16, + 0, 0, 8, 0, 0, 0, + 48, 0, 255, 255, 255, 255, + 255, 255, 79, 117, 116, 112, + 117, 116, 83, 105, 122, 101, + 115, 0, 34, 0, 81, 17, + 20, 16, 0, 0, 8, 0, + 0, 0, 64, 0, 255, 255, + 255, 255, 255, 255, 79, 117, + 116, 112, 117, 116, 83, 116, + 114, 105, 100, 101, 115, 0, + 0, 0, 26, 0, 81, 17, + 21, 16, 0, 0, 8, 0, + 0, 0, 80, 0, 255, 255, + 255, 255, 255, 255, 83, 99, + 97, 108, 101, 0, 0, 0, + 30, 0, 81, 17, 17, 16, + 0, 0, 8, 0, 0, 0, + 84, 0, 255, 255, 255, 255, + 255, 255, 68, 70, 84, 76, + 101, 110, 103, 116, 104, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 255, 255, + 255, 255, 26, 9, 47, 241, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 119, 9, 49, 1, 1, 0, + 0, 0, 13, 0, 20, 142, + 14, 0, 20, 107, 15, 0, + 1, 0, 76, 0, 0, 0, + 32, 0, 0, 0, 44, 0, + 0, 0, 124, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 22, 0, 0, 0, + 25, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, + 132, 10, 0, 0, 32, 0, + 0, 96, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 2, 0, 9, 0, + 24, 16, 0, 0, 0, 0, + 0, 0, 12, 10, 0, 0, + 1, 0, 0, 0, 184, 7, + 140, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 68, 70, + 84, 0, 110, 111, 110, 101, + 0, 0, 0, 0, 45, 186, + 46, 241, 1, 0, 0, 0, + 0, 0, 0, 0, 132, 10, + 0, 0, 32, 0, 0, 96, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 2, 0, 2, 0, 7, 0, + 0, 0, 0, 0, 1, 0, + 255, 255, 255, 255, 0, 0, + 0, 0, 132, 10, 0, 0, + 8, 2, 0, 0, 0, 0, + 0, 0, 255, 255, 255, 255, + 0, 0, 0, 0, 255, 255, + 255, 255, 1, 0, 1, 0, + 0, 0, 1, 0, 0, 0, + 0, 0, 69, 58, 92, 119, + 111, 114, 107, 92, 111, 110, + 110, 120, 114, 117, 110, 116, + 105, 109, 101, 51, 92, 111, + 110, 110, 120, 114, 117, 110, + 116, 105, 109, 101, 92, 99, + 111, 114, 101, 92, 112, 114, + 111, 118, 105, 100, 101, 114, + 115, 92, 100, 109, 108, 92, + 68, 109, 108, 69, 120, 101, + 99, 117, 116, 105, 111, 110, + 80, 114, 111, 118, 105, 100, + 101, 114, 92, 115, 114, 99, + 92, 79, 112, 101, 114, 97, + 116, 111, 114, 115, 92, 83, + 104, 97, 100, 101, 114, 115, + 92, 115, 116, 111, 99, 107, + 104, 97, 109, 46, 104, 108, + 115, 108, 0, 0, 0, 0, + 254, 239, 254, 239, 1, 0, + 0, 0, 1, 0, 0, 0, + 0, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 12, + 0, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 10, 32, + 32, 32, 32, 117, 105, 110, + 116, 52, 32, 79, 117, 116, + 112, 117, 116, 83, 116, 114, + 105, 100, 101, 115, 59, 13, + 10, 32, 32, 32, 32, 102, + 108, 111, 97, 116, 32, 83, + 99, 97, 108, 101, 59, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 32, 68, 70, + 84, 76, 101, 110, 103, 116, + 104, 59, 13, 10, 125, 59, + 13, 10, 13, 10, 47, 47, + 32, 82, 101, 116, 117, 114, + 110, 115, 32, 116, 104, 101, + 32, 105, 110, 100, 105, 99, + 101, 115, 32, 102, 111, 114, + 32, 116, 104, 101, 32, 114, + 101, 97, 108, 32, 97, 110, + 100, 32, 99, 111, 109, 112, + 108, 101, 120, 32, 111, 117, + 116, 112, 117, 116, 32, 117, + 97, 118, 13, 10, 117, 105, + 110, 116, 50, 32, 67, 111, + 109, 112, 117, 116, 101, 68, + 101, 115, 116, 73, 110, 100, + 101, 120, 40, 117, 105, 110, + 116, 32, 105, 110, 100, 101, + 120, 41, 13, 10, 123, 13, + 10, 32, 32, 32, 32, 117, + 105, 110, 116, 50, 32, 100, + 102, 116, 79, 117, 116, 112, + 117, 116, 73, 110, 100, 101, + 120, 32, 61, 32, 117, 105, + 110, 116, 50, 40, 105, 110, + 100, 101, 120, 32, 42, 32, + 79, 117, 116, 112, 117, 116, + 83, 116, 114, 105, 100, 101, + 115, 91, 50, 93, 44, 32, + 48, 41, 59, 13, 10, 32, + 32, 32, 32, 100, 102, 116, + 79, 117, 116, 112, 117, 116, + 73, 110, 100, 101, 120, 46, + 121, 32, 61, 32, 100, 102, + 116, 79, 117, 116, 112, 117, + 116, 73, 110, 100, 101, 120, + 46, 120, 32, 43, 32, 79, + 117, 116, 112, 117, 116, 83, + 116, 114, 105, 100, 101, 115, + 91, 51, 93, 59, 13, 10, + 32, 32, 32, 32, 114, 101, + 116, 117, 114, 110, 32, 100, + 102, 116, 79, 117, 116, 112, + 117, 116, 73, 110, 100, 101, + 120, 59, 13, 10, 125, 13, + 10, 13, 10, 47, 47, 32, + 84, 104, 101, 32, 114, 101, + 116, 117, 114, 110, 101, 100, + 32, 118, 97, 108, 117, 101, + 32, 105, 115, 32, 102, 108, + 111, 97, 116, 50, 44, 32, + 99, 111, 114, 114, 101, 115, + 112, 111, 110, 100, 105, 110, + 103, 32, 116, 111, 32, 116, + 104, 101, 32, 99, 111, 109, + 112, 108, 101, 120, 32, 110, + 117, 109, 98, 101, 114, 32, + 97, 116, 32, 116, 104, 101, + 32, 105, 110, 100, 101, 120, + 13, 10, 102, 108, 111, 97, + 116, 50, 32, 82, 101, 97, + 100, 83, 111, 117, 114, 99, + 101, 86, 97, 108, 117, 101, + 40, 117, 105, 110, 116, 51, + 32, 105, 110, 100, 101, 120, + 41, 13, 10, 123, 13, 10, + 32, 32, 32, 32, 102, 108, + 111, 97, 116, 50, 32, 118, + 97, 108, 117, 101, 32, 61, + 32, 102, 108, 111, 97, 116, + 50, 40, 48, 44, 32, 48, + 41, 59, 13, 10, 13, 10, + 32, 32, 32, 32, 117, 105, + 110, 116, 32, 105, 110, 100, + 101, 120, 82, 101, 97, 108, + 32, 61, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 105, 110, 100, 101, 120, 46, + 120, 32, 42, 32, 73, 110, + 112, 117, 116, 83, 116, 114, + 105, 100, 101, 115, 91, 48, + 93, 32, 43, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 105, 110, 100, 101, 120, + 46, 121, 32, 42, 32, 73, + 110, 112, 117, 116, 83, 116, + 114, 105, 100, 101, 115, 91, + 49, 93, 32, 43, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 105, 110, 100, 101, + 120, 46, 122, 32, 42, 32, + 73, 110, 112, 117, 116, 83, + 116, 114, 105, 100, 101, 115, + 91, 50, 93, 59, 13, 10, + 32, 32, 32, 32, 118, 97, + 108, 117, 101, 46, 120, 32, + 61, 32, 115, 114, 99, 91, + 105, 110, 100, 101, 120, 82, + 101, 97, 108, 93, 59, 13, + 10, 13, 10, 32, 32, 32, + 32, 47, 47, 32, 73, 102, + 32, 114, 101, 97, 108, 32, + 118, 97, 108, 117, 101, 100, + 44, 32, 118, 97, 108, 117, + 101, 46, 121, 32, 105, 115, + 32, 100, 101, 102, 97, 117, + 108, 116, 101, 100, 32, 116, + 111, 32, 48, 13, 10, 32, + 32, 32, 32, 47, 47, 32, + 73, 102, 32, 99, 111, 109, + 112, 108, 101, 120, 32, 118, + 97, 108, 117, 101, 100, 32, + 105, 110, 112, 117, 116, 44, + 32, 97, 115, 115, 105, 103, + 110, 32, 116, 104, 101, 32, + 99, 111, 109, 112, 108, 101, + 120, 32, 112, 97, 114, 116, + 32, 116, 111, 32, 110, 111, + 110, 45, 122, 101, 114, 111, + 46, 46, 46, 13, 10, 32, + 32, 32, 32, 105, 102, 32, + 40, 73, 110, 112, 117, 116, + 83, 105, 122, 101, 115, 91, + 51, 93, 32, 61, 61, 32, + 50, 41, 32, 123, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 105, 110, 100, 101, 120, + 73, 109, 97, 103, 105, 110, + 97, 114, 121, 32, 61, 32, + 105, 110, 100, 101, 120, 82, + 101, 97, 108, 32, 43, 32, + 73, 110, 112, 117, 116, 83, + 116, 114, 105, 100, 101, 115, + 91, 51, 93, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 118, 97, 108, 117, + 101, 46, 121, 32, 61, 32, + 115, 114, 99, 91, 105, 110, + 100, 101, 120, 73, 109, 97, + 103, 105, 110, 97, 114, 121, + 93, 59, 13, 10, 32, 32, + 32, 32, 125, 13, 10, 13, + 10, 32, 32, 32, 32, 114, + 101, 116, 117, 114, 110, 32, + 118, 97, 108, 117, 101, 59, + 13, 10, 125, 13, 10, 13, + 10, 117, 105, 110, 116, 51, + 32, 68, 101, 99, 111, 109, + 112, 111, 115, 101, 73, 110, + 100, 101, 120, 40, 117, 105, + 110, 116, 32, 105, 110, 100, + 101, 120, 41, 13, 10, 123, + 13, 10, 32, 32, 32, 32, + 117, 105, 110, 116, 32, 116, + 101, 109, 112, 32, 61, 32, + 105, 110, 100, 101, 120, 32, + 37, 32, 40, 79, 117, 116, + 112, 117, 116, 83, 105, 122, + 101, 115, 91, 49, 93, 32, + 42, 32, 79, 117, 116, 112, + 117, 116, 83, 105, 122, 101, + 115, 91, 50, 93, 41, 59, + 13, 10, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 51, 32, 105, 100, 120, 32, + 61, 32, 117, 105, 110, 116, + 51, 40, 48, 44, 32, 48, + 44, 32, 48, 41, 59, 13, + 10, 32, 32, 32, 32, 105, + 100, 120, 46, 120, 32, 61, + 32, 105, 110, 100, 101, 120, + 32, 47, 32, 40, 79, 117, + 116, 112, 117, 116, 83, 105, + 122, 101, 115, 91, 49, 93, + 32, 42, 32, 79, 117, 116, + 112, 117, 116, 83, 105, 122, + 101, 115, 91, 50, 93, 41, + 59, 13, 10, 32, 32, 32, + 32, 105, 100, 120, 46, 121, + 32, 61, 32, 116, 101, 109, + 112, 32, 47, 32, 79, 117, + 116, 112, 117, 116, 83, 105, + 122, 101, 115, 91, 50, 93, + 59, 32, 47, 47, 32, 84, + 104, 105, 115, 32, 99, 111, + 114, 114, 101, 115, 112, 111, + 110, 100, 115, 32, 116, 111, + 32, 116, 104, 101, 32, 115, + 49, 39, 116, 104, 32, 101, + 108, 101, 109, 101, 110, 116, + 32, 111, 102, 32, 116, 104, + 101, 32, 100, 102, 116, 13, + 10, 32, 32, 32, 32, 105, + 100, 120, 46, 122, 32, 61, + 32, 116, 101, 109, 112, 32, + 37, 32, 79, 117, 116, 112, + 117, 116, 83, 105, 122, 101, + 115, 91, 50, 93, 59, 13, + 10, 32, 32, 32, 32, 114, + 101, 116, 117, 114, 110, 32, + 105, 100, 120, 59, 13, 10, + 125, 13, 10, 13, 10, 91, + 110, 117, 109, 116, 104, 114, + 101, 97, 100, 115, 40, 54, + 52, 44, 32, 49, 44, 32, + 49, 41, 93, 13, 10, 118, + 111, 105, 100, 32, 68, 70, + 84, 40, 117, 105, 110, 116, + 51, 32, 100, 116, 105, 100, + 32, 58, 32, 83, 86, 95, + 68, 105, 115, 112, 97, 116, + 99, 104, 84, 104, 114, 101, + 97, 100, 73, 100, 41, 13, + 10, 123, 13, 10, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 105, 110, 100, 101, 120, + 32, 61, 32, 83, 116, 97, + 114, 116, 73, 110, 100, 101, + 120, 32, 43, 32, 100, 116, + 105, 100, 46, 120, 59, 13, + 10, 32, 32, 32, 32, 105, + 102, 32, 40, 105, 110, 100, + 101, 120, 32, 60, 32, 69, + 108, 101, 109, 101, 110, 116, + 67, 111, 117, 110, 116, 41, + 13, 10, 32, 32, 32, 32, + 123, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 117, + 105, 110, 116, 32, 104, 97, + 108, 102, 84, 111, 116, 97, + 108, 68, 70, 84, 76, 101, + 110, 103, 116, 104, 32, 61, + 32, 68, 70, 84, 76, 101, + 110, 103, 116, 104, 32, 47, + 32, 50, 59, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 117, 105, 110, 116, 32, + 78, 32, 61, 32, 49, 32, + 60, 60, 32, 68, 70, 84, + 73, 116, 101, 114, 97, 116, + 105, 111, 110, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 117, 105, 110, 116, + 32, 104, 97, 108, 102, 78, + 32, 61, 32, 49, 32, 60, + 60, 32, 40, 68, 70, 84, + 73, 116, 101, 114, 97, 116, + 105, 111, 110, 32, 45, 32, + 49, 41, 59, 13, 10, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 47, 47, 32, + 71, 101, 116, 32, 105, 110, + 112, 117, 116, 32, 101, 118, + 101, 110, 32, 97, 110, 100, + 32, 111, 100, 100, 32, 105, + 110, 100, 105, 99, 101, 115, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 47, 47, + 32, 68, 101, 99, 111, 109, + 112, 111, 115, 101, 32, 116, + 104, 101, 32, 99, 117, 114, + 114, 101, 110, 116, 32, 105, + 110, 100, 101, 120, 32, 105, + 110, 116, 111, 32, 105, 116, + 115, 32, 108, 111, 99, 97, + 116, 105, 111, 110, 32, 105, + 110, 32, 116, 104, 101, 32, + 112, 97, 99, 107, 101, 100, + 32, 116, 101, 110, 115, 111, + 114, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 117, + 105, 110, 116, 50, 32, 105, + 110, 112, 117, 116, 69, 118, + 101, 110, 79, 100, 100, 73, + 110, 100, 101, 120, 80, 97, + 105, 114, 32, 61, 32, 117, + 105, 110, 116, 50, 40, 48, + 44, 32, 48, 41, 59, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 117, 105, 110, + 116, 51, 32, 105, 100, 120, + 32, 61, 32, 68, 101, 99, + 111, 109, 112, 111, 115, 101, + 73, 110, 100, 101, 120, 40, + 105, 110, 100, 101, 120, 41, + 59, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 105, + 110, 112, 117, 116, 69, 118, + 101, 110, 79, 100, 100, 73, + 110, 100, 101, 120, 80, 97, + 105, 114, 46, 120, 32, 61, + 32, 40, 105, 100, 120, 46, + 121, 32, 62, 62, 32, 68, + 70, 84, 73, 116, 101, 114, + 97, 116, 105, 111, 110, 41, + 32, 42, 32, 104, 97, 108, + 102, 78, 32, 43, 32, 40, + 105, 100, 120, 46, 121, 32, + 37, 32, 104, 97, 108, 102, + 78, 41, 59, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 105, 110, 112, 117, 116, + 69, 118, 101, 110, 79, 100, + 100, 73, 110, 100, 101, 120, + 80, 97, 105, 114, 46, 121, + 32, 61, 32, 105, 110, 112, + 117, 116, 69, 118, 101, 110, + 79, 100, 100, 73, 110, 100, + 101, 120, 80, 97, 105, 114, + 46, 120, 32, 43, 32, 104, + 97, 108, 102, 84, 111, 116, + 97, 108, 68, 70, 84, 76, + 101, 110, 103, 116, 104, 59, + 13, 10, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 47, 47, 32, 67, 114, 101, + 97, 116, 101, 32, 102, 117, + 108, 108, 32, 105, 110, 100, + 101, 120, 32, 102, 111, 114, + 32, 101, 118, 101, 110, 32, + 97, 110, 100, 32, 111, 100, + 100, 32, 118, 97, 108, 117, + 101, 115, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 117, 105, 110, 116, 51, 32, + 105, 110, 112, 117, 116, 69, + 118, 101, 110, 73, 100, 120, + 32, 61, 32, 117, 105, 110, + 116, 51, 40, 105, 100, 120, + 46, 120, 44, 32, 105, 110, + 112, 117, 116, 69, 118, 101, + 110, 79, 100, 100, 73, 110, + 100, 101, 120, 80, 97, 105, + 114, 46, 120, 44, 32, 105, + 100, 120, 46, 122, 41, 59, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 117, 105, + 110, 116, 51, 32, 105, 110, + 112, 117, 116, 79, 100, 100, + 73, 100, 120, 32, 61, 32, + 117, 105, 110, 116, 51, 40, + 105, 100, 120, 46, 120, 44, + 32, 105, 110, 112, 117, 116, + 69, 118, 101, 110, 79, 100, + 100, 73, 110, 100, 101, 120, + 80, 97, 105, 114, 46, 121, + 44, 32, 105, 100, 120, 46, + 122, 41, 59, 13, 10, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 47, 47, 32, + 82, 101, 97, 100, 32, 105, + 110, 112, 117, 116, 32, 101, + 118, 101, 110, 32, 97, 110, + 100, 32, 111, 100, 100, 32, + 118, 97, 108, 117, 101, 115, + 13, 10, 32, 32, 32, 32, + 32, 32, 32, 32, 102, 108, + 111, 97, 116, 50, 32, 105, + 110, 112, 117, 116, 69, 118, + 101, 110, 86, 97, 108, 117, + 101, 32, 61, 32, 82, 101, + 97, 100, 83, 111, 117, 114, + 99, 101, 86, 97, 108, 117, + 101, 40, 105, 110, 112, 117, + 116, 69, 118, 101, 110, 73, + 100, 120, 41, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 102, 108, 111, 97, + 116, 50, 32, 105, 110, 112, + 117, 116, 79, 100, 100, 86, + 97, 108, 117, 101, 32, 61, + 32, 82, 101, 97, 100, 83, + 111, 117, 114, 99, 101, 86, + 97, 108, 117, 101, 40, 105, + 110, 112, 117, 116, 79, 100, + 100, 73, 100, 120, 41, 59, + 13, 10, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 47, 47, 32, 67, 114, 101, + 97, 116, 101, 32, 99, 111, + 101, 102, 102, 105, 99, 105, + 101, 110, 116, 13, 10, 32, + 32, 32, 32, 32, 32, 32, + 32, 47, 47, 32, 119, 40, + 107, 44, 32, 78, 41, 32, + 61, 32, 101, 94, 40, 105, + 42, 50, 42, 112, 105, 32, + 42, 32, 107, 32, 47, 32, + 78, 41, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 117, 105, 110, 116, 32, 107, + 32, 61, 32, 105, 100, 120, + 46, 121, 32, 37, 32, 78, + 59, 13, 10, 32, 32, 32, + 32, 32, 32, 32, 32, 115, + 116, 97, 116, 105, 99, 32, + 99, 111, 110, 115, 116, 32, + 102, 108, 111, 97, 116, 32, + 80, 73, 32, 61, 32, 51, + 46, 49, 52, 49, 53, 57, + 50, 54, 53, 102, 59, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 115, 116, 97, + 116, 105, 99, 32, 99, 111, + 110, 115, 116, 32, 102, 108, + 111, 97, 116, 32, 84, 65, + 85, 32, 61, 32, 80, 73, + 32, 42, 32, 50, 59, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 98, 111, 111, + 108, 32, 105, 115, 73, 110, + 118, 101, 114, 115, 101, 32, + 61, 32, 73, 115, 73, 110, + 118, 101, 114, 115, 101, 32, + 61, 61, 32, 49, 59, 13, + 10, 32, 32, 32, 32, 32, + 32, 32, 32, 99, 111, 110, + 115, 116, 32, 102, 108, 111, + 97, 116, 32, 105, 110, 118, + 101, 114, 115, 101, 77, 117, + 108, 116, 105, 112, 108, 105, + 101, 114, 32, 61, 32, 105, + 115, 73, 110, 118, 101, 114, + 115, 101, 32, 63, 32, 49, + 46, 102, 32, 58, 32, 45, + 49, 46, 102, 59, 13, 10, + 32, 32, 32, 32, 32, 32, + 32, 32, 102, 108, 111, 97, + 116, 32, 116, 104, 101, 116, + 97, 32, 61, 32, 105, 110, + 118, 101, 114, 115, 101, 77, + 117, 108, 116, 105, 112, 108, + 105, 101, 114, 32, 42, 32, + 84, 65, 85, 32, 42, 32, + 40, 102, 108, 111, 97, 116, + 41, 107, 32, 47, 32, 40, + 102, 108, 111, 97, 116, 41, + 78, 59, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 102, 108, 111, 97, 116, 50, + 32, 119, 32, 61, 32, 102, + 108, 111, 97, 116, 50, 40, + 99, 111, 115, 40, 116, 104, + 101, 116, 97, 41, 44, 32, + 115, 105, 110, 40, 116, 104, + 101, 116, 97, 41, 41, 59, + 13, 10, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 117, 105, 110, 116, 50, 32, + 111, 117, 116, 112, 117, 116, + 73, 110, 100, 101, 120, 32, + 61, 32, 67, 111, 109, 112, + 117, 116, 101, 68, 101, 115, + 116, 73, 110, 100, 101, 120, + 40, 105, 110, 100, 101, 120, + 41, 59, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 100, 115, 116, 91, 111, 117, + 116, 112, 117, 116, 73, 110, + 100, 101, 120, 46, 120, 93, + 32, 61, 32, 83, 99, 97, + 108, 101, 32, 42, 32, 40, + 105, 110, 112, 117, 116, 69, + 118, 101, 110, 86, 97, 108, + 117, 101, 46, 120, 32, 43, + 32, 40, 119, 46, 120, 32, + 42, 32, 105, 110, 112, 117, + 116, 79, 100, 100, 86, 97, + 108, 117, 101, 46, 120, 32, + 45, 32, 119, 46, 121, 32, + 42, 32, 105, 110, 112, 117, + 116, 79, 100, 100, 86, 97, + 108, 117, 101, 46, 121, 41, + 41, 59, 13, 10, 32, 32, + 32, 32, 32, 32, 32, 32, + 100, 115, 116, 91, 111, 117, + 116, 112, 117, 116, 73, 110, + 100, 101, 120, 46, 121, 93, + 32, 61, 32, 83, 99, 97, + 108, 101, 32, 42, 32, 40, + 105, 110, 112, 117, 116, 69, + 118, 101, 110, 86, 97, 108, + 117, 101, 46, 121, 32, 43, + 32, 40, 119, 46, 120, 32, + 42, 32, 105, 110, 112, 117, + 116, 79, 100, 100, 86, 97, + 108, 117, 101, 46, 121, 32, + 43, 32, 119, 46, 121, 32, + 42, 32, 105, 110, 112, 117, + 116, 79, 100, 100, 86, 97, + 108, 117, 101, 46, 120, 41, + 41, 59, 13, 10, 32, 32, + 32, 32, 125, 13, 10, 125, + 13, 10, 0, 7, 0, 0, + 0, 1, 0, 0, 0, 110, + 0, 0, 0, 111, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 220, 0, 0, + 0, 0, 0, 0, 0, 4, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 148, 46, + 49, 1, 68, 252, 16, 99, + 1, 0, 0, 0, 4, 192, + 184, 18, 49, 85, 209, 71, + 177, 201, 131, 74, 11, 196, + 189, 67, 154, 0, 0, 0, + 47, 76, 105, 110, 107, 73, + 110, 102, 111, 0, 47, 110, + 97, 109, 101, 115, 0, 47, + 115, 114, 99, 47, 104, 101, + 97, 100, 101, 114, 98, 108, + 111, 99, 107, 0, 47, 115, + 114, 99, 47, 102, 105, 108, + 101, 115, 47, 101, 58, 92, + 119, 111, 114, 107, 92, 111, + 110, 110, 120, 114, 117, 110, + 116, 105, 109, 101, 51, 92, + 111, 110, 110, 120, 114, 117, + 110, 116, 105, 109, 101, 92, + 99, 111, 114, 101, 92, 112, + 114, 111, 118, 105, 100, 101, + 114, 115, 92, 100, 109, 108, + 92, 100, 109, 108, 101, 120, + 101, 99, 117, 116, 105, 111, + 110, 112, 114, 111, 118, 105, + 100, 101, 114, 92, 115, 114, + 99, 92, 111, 112, 101, 114, + 97, 116, 111, 114, 115, 92, + 115, 104, 97, 100, 101, 114, + 115, 92, 115, 116, 111, 99, + 107, 104, 97, 109, 46, 104, + 108, 115, 108, 0, 4, 0, + 0, 0, 6, 0, 0, 0, + 1, 0, 0, 0, 30, 0, + 0, 0, 0, 0, 0, 0, + 17, 0, 0, 0, 7, 0, + 0, 0, 34, 0, 0, 0, + 8, 0, 0, 0, 10, 0, + 0, 0, 6, 0, 0, 0, + 0, 0, 0, 0, 5, 0, + 0, 0, 0, 0, 0, 0, + 220, 81, 51, 1, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 32, 0, + 0, 0, 246, 0, 0, 0, + 120, 1, 0, 0, 131, 1, + 0, 0, 144, 0, 0, 0, + 0, 0, 0, 0, 243, 13, + 0, 0, 128, 0, 0, 0, + 230, 12, 0, 0, 92, 26, + 0, 0, 96, 0, 0, 0, + 20, 0, 0, 0, 40, 0, + 0, 0, 176, 2, 0, 0, + 44, 0, 0, 0, 144, 1, + 0, 0, 3, 0, 0, 0, + 45, 0, 0, 0, 31, 0, + 0, 0, 38, 0, 0, 0, + 6, 0, 0, 0, 15, 0, + 0, 0, 39, 0, 0, 0, + 40, 0, 0, 0, 41, 0, + 0, 0, 42, 0, 0, 0, + 43, 0, 0, 0, 44, 0, + 0, 0, 16, 0, 0, 0, + 8, 0, 0, 0, 9, 0, + 0, 0, 10, 0, 0, 0, + 11, 0, 0, 0, 12, 0, + 0, 0, 13, 0, 0, 0, + 14, 0, 0, 0, 17, 0, + 0, 0, 18, 0, 0, 0, + 19, 0, 0, 0, 20, 0, + 0, 0, 21, 0, 0, 0, + 22, 0, 0, 0, 23, 0, + 0, 0, 24, 0, 0, 0, + 25, 0, 0, 0, 26, 0, + 0, 0, 27, 0, 0, 0, + 28, 0, 0, 0, 29, 0, + 0, 0, 30, 0, 0, 0, + 7, 0, 0, 0, 32, 0, + 0, 0, 33, 0, 0, 0, + 34, 0, 0, 0, 35, 0, + 0, 0, 37, 0, 0, 0, + 36, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 46, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 +}; diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp index b39d886177..7ddbd39398 100644 --- a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/OperatorRegistration.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "precomp.h" +#include "DmlDFT.h" #include "OperatorRegistration.h" #include "core/providers/dml/OperatorAuthorHelper/MLOperatorAuthorHelper.h" #include "core/providers/dml/OperatorAuthorHelper/OperatorVersions.h" @@ -829,6 +830,8 @@ void RegisterDmlOperators(IMLOperatorRegistry* registry) static_cast(information.requiredConstantCpuInputs.second) )); } + + GpuDFTOperatorFactory::RegisterDFTKernel(registry); } } // namespace Dml diff --git a/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/Shaders/stockham.hlsl b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/Shaders/stockham.hlsl new file mode 100644 index 0000000000..deac28be58 --- /dev/null +++ b/onnxruntime/core/providers/dml/DmlExecutionProvider/src/Operators/Shaders/stockham.hlsl @@ -0,0 +1,97 @@ +RWStructuredBuffer src : register(u0); +RWStructuredBuffer dst : register(u1); + +cbuffer Constants +{ + uint StartIndex; + uint ElementCount; + uint DFTIteration; + uint IsInverse; + uint4 InputSizes; + uint4 InputStrides; + uint4 OutputSizes; + uint4 OutputStrides; + float Scale; + uint DFTLength; +}; + +// Returns the indices for the real and complex output uav +uint2 ComputeDestIndex(uint index) +{ + uint2 dftOutputIndex = uint2(index * OutputStrides[2], 0); + dftOutputIndex.y = dftOutputIndex.x + OutputStrides[3]; + return dftOutputIndex; +} + +// The returned value is float2, corresponding to the complex number at the index +float2 ReadSourceValue(uint3 index) +{ + float2 value = float2(0, 0); + + uint indexReal = + index.x * InputStrides[0] + + index.y * InputStrides[1] + + index.z * InputStrides[2]; + value.x = src[indexReal]; + + // If real valued, value.y is defaulted to 0 + // If complex valued input, assign the complex part to non-zero... + if (InputSizes[3] == 2) { + uint indexImaginary = indexReal + InputStrides[3]; + value.y = src[indexImaginary]; + } + + return value; +} + +uint3 DecomposeIndex(uint index) +{ + uint temp = index % (OutputSizes[1] * OutputSizes[2]); + + uint3 idx = uint3(0, 0, 0); + idx.x = index / (OutputSizes[1] * OutputSizes[2]); + idx.y = temp / OutputSizes[2]; // This corresponds to the s1'th element of the dft + idx.z = temp % OutputSizes[2]; + return idx; +} + +[numthreads(64, 1, 1)] +void DFT(uint3 dtid : SV_DispatchThreadId) +{ + uint index = StartIndex + dtid.x; + if (index < ElementCount) + { + uint halfTotalDFTLength = DFTLength / 2; + uint N = 1 << DFTIteration; + uint halfN = 1 << (DFTIteration - 1); + + // Get input even and odd indices + // Decompose the current index into its location in the packed tensor + uint2 inputEvenOddIndexPair = uint2(0, 0); + uint3 idx = DecomposeIndex(index); + inputEvenOddIndexPair.x = (idx.y >> DFTIteration) * halfN + (idx.y % halfN); + inputEvenOddIndexPair.y = inputEvenOddIndexPair.x + halfTotalDFTLength; + + // Create full index for even and odd values + uint3 inputEvenIdx = uint3(idx.x, inputEvenOddIndexPair.x, idx.z); + uint3 inputOddIdx = uint3(idx.x, inputEvenOddIndexPair.y, idx.z); + + // Read input even and odd values + float2 inputEvenValue = ReadSourceValue(inputEvenIdx); + float2 inputOddValue = ReadSourceValue(inputOddIdx); + + // Create coefficient + // w(k, N) = e^(i*2*pi * k / N) + uint k = idx.y % N; + static const float PI = 3.14159265f; + static const float TAU = PI * 2; + bool isInverse = IsInverse == 1; + const float inverseMultiplier = isInverse ? 1.f : -1.f; + float theta = inverseMultiplier * TAU * (float)k / (float)N; + float2 w = float2(cos(theta), sin(theta)); + + uint2 outputIndex = ComputeDestIndex(index); + dst[outputIndex.x] = Scale * (inputEvenValue.x + (w.x * inputOddValue.x - w.y * inputOddValue.y)); + dst[outputIndex.y] = Scale * (inputEvenValue.y + (w.x * inputOddValue.y + w.y * inputOddValue.x)); + } +} diff --git a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp index bd096a20e0..9d3652f99c 100644 --- a/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp +++ b/onnxruntime/core/providers/dml/OperatorAuthorHelper/OperatorHelper.cpp @@ -851,7 +851,7 @@ namespace OperatorHelper int paddings = gsl::narrow_cast((inputDimensions[i + dimOffset] - 1) * stride + windowSize - m_outputShapes[0].GetShape()[i + dimOffset]); paddings = std::max(0, paddings); - m_kernel.startPadding[i] = m_kernel.autoPadSameUpper ? (paddings + 1) / 2 : paddings / 2; + m_kernel.startPadding[i] = m_kernel.autoPadSameUpper ? paddings / 2 : (paddings + 1) / 2; m_kernel.endPadding[i] = paddings - m_kernel.startPadding[i]; } } @@ -2047,8 +2047,14 @@ namespace OperatorHelper m_axes = kernelInformation.GetAttributes().GetOptionalAttributeVectorInt32(AttrName::Axes); } std::vector inputDimensions = shapeInformation.GetInputTensorShape(0); + HandleNegativeAxes(/*inout*/ m_axes, gsl::narrow_cast(inputDimensions.size())); std::sort(m_axes.begin(), m_axes.end()); + if (m_axes.empty()) + { + m_axes.resize(inputDimensions.size()); + std::iota(m_axes.begin(), m_axes.end(), 0u); + } } std::vector SqueezeHelper::GetOutputShapes(const MLShapeInferenceContext& shapeInfo) const diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.cc index cbd92f4a21..cde20fdaca 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.cc @@ -25,6 +25,18 @@ void DnnlDequantizeLinear::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& // Check if scale and zp are scalars bool isScalar = sp.IsScalar(node.Input(IN_X_SCALE)); + // Check if zp is needed + bool isZeroPointUseful = false; + if (node.Input(IN_X_ZERO_POINT).Exists()) { + // If zp exists then it's needed + isZeroPointUseful = true; + // If it's constant then we can evaluate if zp == 0 + if (node.Input(IN_X_ZERO_POINT).IsConstant()) { + // if zp == 0 then isZeroPointUseful = false; else isZeroPointUseful = true + auto mem = sp.GetMemory(node.Input(IN_X_ZERO_POINT)); + isZeroPointUseful = isZeroPointNonZero(&mem); + } + } // Get the x and scale mem auto x_mem = sp.GetMemory(node.Input(IN_X)); @@ -43,9 +55,13 @@ void DnnlDequantizeLinear::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& if (axis < 0) { axis += x_dims; } - // If scale is a vector, add padding for broadcasting - if (!isScalar) { - Padd(&x_scale_md, static_cast(axis + 1), x_dims); + // Prepare the scale to prevent broacasting errors + if (isScalar) { + // For scalar scale + Padd(&x_scale_md, x_dims, false); + } else { + // For N-D scale + Padd(&x_scale_md, static_cast(axis) + 1, x_dims); } // Create dst mem @@ -53,8 +69,7 @@ void DnnlDequantizeLinear::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& dnnl::memory dst_mem; // If zero point exists and we are NOT dequantizing int32, then substract zp from x and scale - if (node.Input(IN_X_ZERO_POINT).Exists() && - (x_mem.get_desc().data_type() != dnnl::memory::data_type::s32)) { + if (isZeroPointUseful && (x_mem.get_desc().data_type() != dnnl::memory::data_type::s32)) { // Get Zero point auto x_zp_mem = sp.GetMemory(node.Input(IN_X_ZERO_POINT)); // Get mds for operands @@ -66,16 +81,18 @@ void DnnlDequantizeLinear::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& Padd(&x_zp_md, x_dims, false); } else { // For N-D zp - Padd(&x_zp_md, static_cast(axis) + 1, x_md.dims().size()); + Padd(&x_zp_md, static_cast(axis) + 1, x_dims); } // Create binary desc auto binary_d = dnnl::binary::desc(dnnl::algorithm::binary_sub, x_md, x_zp_md, dst_md); // Add post op scale - dnnl::post_ops binary_ops; dnnl::primitive_attr binary_attr; - binary_ops.append_binary(dnnl::algorithm::binary_mul, x_scale_md); - binary_attr.set_post_ops(binary_ops); + { + dnnl::post_ops binary_ops; + binary_ops.append_binary(dnnl::algorithm::binary_mul, x_scale_md); + binary_attr.set_post_ops(binary_ops); + } // Add post op to scale result auto binary_pd = dnnl::binary::primitive_desc(binary_d, binary_attr, dnnl_engine); // Move to GPU if available @@ -99,7 +116,6 @@ void DnnlDequantizeLinear::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& dst_mem = dnnl::memory(binary_pd.dst_desc(), dnnl_engine); auto binary_prim = dnnl::binary(binary_pd); - // We recycle the x_mem sp.AddPrimitive(binary_prim, {{DNNL_ARG_SRC_0, x_mem}, {DNNL_ARG_SRC_1, x_scale_mem}, {DNNL_ARG_DST, dst_mem}}); @@ -113,6 +129,25 @@ void DnnlDequantizeLinear::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& } } +bool DnnlDequantizeLinear::isZeroPointNonZero(dnnl::memory* zp_mem) { + // Because zp will always be int8, uint8 or int32, this cast is always valid + auto zp_data = static_cast(zp_mem->get_data_handle()); + // Adjust the iteration num + auto topline = zp_mem->get_desc().dims().size(); + if (zp_mem->get_desc().data_type() == dnnl::memory::data_type::s32) { + topline *= 4; + } + // ZP is either a scalar or a 1-D vector so iterate over all the dimensions + // and search for a zp != 0 + for (size_t i = 0; i < topline; ++i) { + if (zp_data[i] != 0) { + return true; + } + } + // If ZP is full of zeros then it is not needed + return false; +} + void DnnlDequantizeLinear::Padd(dnnl::memory::desc* target_md, size_t front_pad, size_t back_pad) { // Pads an input to broadcast the op correctly auto target_dims = target_md->dims(); diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.h b/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.h index 3609e23214..095c5f821e 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.h +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_dequantizelinear.h @@ -24,6 +24,7 @@ class DnnlDequantizeLinear { void CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& node); private: + bool isZeroPointNonZero(dnnl::memory* zp_mem); int64_t GetAxis(DnnlNode& node, size_t x_dims); void Padd(dnnl::memory::desc* target, size_t front_pad, size_t back_pad); void ValidateDims(DnnlSubgraphPrimitive& sp, DnnlNode& node); diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul.cc index b3fe471540..49b7094559 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul.cc @@ -18,7 +18,7 @@ DnnlMatMul::DnnlMatMul() {} // "MatMulPostOps" is a OneDNN only fusion of MatMul and upto 32 elementwise or binary ops. // See dnnl_subgraph_transformer.cc MatMulBinaryEltwise(...). // "FusedMatMul" is a ContribOperator defined here: -// https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md#com.microsoft.FusedMatMul +// https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.FusedMatMul // Depending on its attributes "FusedMatMul" can transpose eather input to the MatMul and scale the resulting output void DnnlMatMul::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& node) { std::unordered_set binary_ops = {"Add", "Div", "Mul", "Sub"}; diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.cc index 5edc03cd27..8e84968da2 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.cc @@ -9,7 +9,7 @@ namespace ort_dnnl { DnnlTensor DnnlNode::empty_tensor_ = DnnlTensor(""); -DnnlTensor::DnnlTensor(const NodeArg* arg) { +DnnlTensor::DnnlTensor(const NodeArg* arg, bool isConstantInitializer) { if (!arg || !arg->Exists()) { tensor_name_ = ""; } else { @@ -20,6 +20,7 @@ DnnlTensor::DnnlTensor(const NodeArg* arg) { arg_type_ = arg->Type(); arg_type_proto_ = ONNX_NAMESPACE::TypeProto::Create(); arg_type_proto_->copy_from(arg->TypeAsProto()); + isConstant_ = isConstantInitializer; } DnnlTensor::DnnlTensor(std::string name) { @@ -124,6 +125,10 @@ bool DnnlTensor::IsDynamic() { return false; } +bool DnnlTensor::IsConstant() { + return isConstant_; +} + bool DnnlTensor::Exists() { return !(tensor_name_ == ""); } @@ -355,7 +360,9 @@ void DnnlSubgraph::Build(const GraphViewer& graph_viewer) { for (auto input : node->InputDefs()) { if (input && input->Exists() && input->Name() != "") { if (!dnnl_tensors_.count(input->Name())) { - dnnl_tensors_[input->Name()] = std::make_unique(input); + dnnl_tensors_[input->Name()] = + std::make_unique(input, + graph_viewer.IsConstantInitializer(input->Name(), true)); } dnnl_tensors_[input->Name()]->AddConsumer(DnnlNodeArg(dnnl_node, index, false)); inputs.push_back(dnnl_tensors_[input->Name()].get()); diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.h b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.h index 35df0d9789..7cdda8cddb 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.h +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph.h @@ -36,16 +36,18 @@ class DnnlNodeArg { class DnnlTensor { public: - DnnlTensor(const NodeArg* arg); + DnnlTensor(const NodeArg* arg, bool isConstantInitializer = false); DnnlTensor(std::string name); DnnlTensor() = default; std::string Name() const; dnnl::memory::dims Dim() const; dnnl::memory::data_type Type() const; dnnl::memory::format_tag Format(); - //check whether the tensor is dynamic, e.g. contains unspecified dimension + // Check whether the tensor is dynamic, e.g. contains unspecified dimension bool IsDynamic(); - //check whether the tensor exsits for optional input output + // Check whether the tensor is constant initializer + bool IsConstant(); + // Check whether the tensor exsits for optional input output bool Exists(); std::vector& GetConsumers() { return consumers_; }; DnnlNodeArg& GetProducer() { return producer_; }; @@ -64,6 +66,7 @@ class DnnlTensor { //a tensor can have no producer (input.initializer) or no consumer (output for subgraph) DnnlNodeArg producer_; std::vector consumers_; + bool isConstant_; }; class DnnlNode { diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc index 4c85fa9790..ca00a9c3cb 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc @@ -197,7 +197,7 @@ void DnnlSubgraphPrimitive::AddKernels() { DnnlLrn().CreatePrimitive(*this, node); // MatMulPostOps is a OneDNN only fusion of MatMul and upto 32 elementwise or binary ops // FusedMatMul is a ContribOperator defined here: - // https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md#com.microsoft.FusedMatMul + // https://github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.FusedMatMul } else if (node.OpType() == "MatMul" || node.OpType() == "MatMulPostOps" || node.OpType() == "FusedMatMul") { DnnlMatMul().CreatePrimitive(*this, node); } else if (node.OpType() == "MatMulInteger" || node.OpType() == "MatMulIntegerPostOps") { diff --git a/onnxruntime/core/providers/get_execution_providers.cc b/onnxruntime/core/providers/get_execution_providers.cc index e97c95397f..8cce064d64 100644 --- a/onnxruntime/core/providers/get_execution_providers.cc +++ b/onnxruntime/core/providers/get_execution_providers.cc @@ -63,14 +63,6 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] = true, #else false, -#endif - }, - { - kNupharExecutionProvider, -#ifdef USE_NUPHAR - true, -#else - false, #endif }, { diff --git a/onnxruntime/core/providers/migraphx/migraphx_call.cc b/onnxruntime/core/providers/migraphx/migraphx_call.cc index f42bb99a67..067227f7c6 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_call.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_call.cc @@ -28,7 +28,8 @@ const char* RocmErrString(hipError_t x) { } template -bool RocmCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { +std::conditional_t RocmCall( + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg) { if (retCode != successCode) { try { char hostname[HOST_NAME_MAX]; @@ -42,25 +43,28 @@ bool RocmCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRT libName, (int)retCode, RocmErrString(retCode), currentHipDevice, hostname, exprString, msg); - if (THRW) { + if constexpr (THRW) { // throw an exception with the error info ORT_THROW(str); } else { LOGS_DEFAULT(ERROR) << str; + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, str); } } catch (const std::exception& e) { // catch, log, and rethrow since HIP code sometimes hangs in destruction, so we'd never get to see the error - if (THRW) { + if constexpr (THRW) { ORT_THROW(e.what()); } else { LOGS_DEFAULT(ERROR) << e.what(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, e.what()); } } - return false; } - return true; + if constexpr (!THRW) { + return Status::OK(); + } } -template bool RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); -template bool RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); +template Status RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); +template void RocmCall(hipError_t retCode, const char* exprString, const char* libName, hipError_t successCode, const char* msg); } // namespace onnxruntime diff --git a/onnxruntime/core/providers/migraphx/migraphx_call.h b/onnxruntime/core/providers/migraphx/migraphx_call.h index 3cf90bf7a6..bbcfc4c3c9 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_call.h +++ b/onnxruntime/core/providers/migraphx/migraphx_call.h @@ -13,9 +13,10 @@ namespace onnxruntime { // ----------------------------------------------------------------------- template -bool RocmCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); +std::conditional_t RocmCall( + ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg = ""); #define HIP_CALL(expr) (RocmCall((expr), #expr, "HIP", hipSuccess)) #define HIP_CALL_THROW(expr) (RocmCall((expr), #expr, "HIP", hipSuccess)) -} \ No newline at end of file +} diff --git a/onnxruntime/core/providers/migraphx/migraphx_execution_provider_info.cc b/onnxruntime/core/providers/migraphx/migraphx_execution_provider_info.cc index 6571593499..0161ca72d9 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_execution_provider_info.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_execution_provider_info.cc @@ -27,9 +27,7 @@ MIGraphXExecutionProviderInfo MIGraphXExecutionProviderInfo::FromProviderOptions [&info](const std::string& value_str) -> Status { ORT_RETURN_IF_ERROR(ParseStringWithClassicLocale(value_str, info.device_id)); int num_devices{}; - ORT_RETURN_IF_NOT( - HIP_CALL(hipGetDeviceCount(&num_devices)), - "hipGetDeviceCount() failed."); + ORT_RETURN_IF_ERROR(HIP_CALL(hipGetDeviceCount(&num_devices))); ORT_RETURN_IF_NOT( 0 <= info.device_id && info.device_id < num_devices, "Invalid device ID: ", info.device_id, diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index c5ddfce065..09058eaad1 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -18,6 +18,7 @@ #include "core/providers/nnapi/nnapi_builtin/builders/op_support_checker.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" +#include "core/optimizer/initializer.h" namespace onnxruntime { namespace nnapi { @@ -193,33 +194,19 @@ common::Status GetQuantizationScaleAndZeroPoint( "NodeArg: ", io_def.node_arg.Name(), " is not quantized"); } - const auto unpack_tensor = [&model_path](const InitializedTensorSet& initializers, - const std::string& name, std::vector& unpacked_tensor) { - const auto& tensor = *initializers.at(name); - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_path, unpacked_tensor)); - return Status::OK(); - }; - const auto& quant_param = *io_def.quant_param; { // get the scale - std::vector unpacked_tensor; const auto& name = quant_param.scale.Name(); - ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor)); + Initializer unpacked_tensor(*initializers.at(name), model_path); // The scale should be one or more floats - ORT_RETURN_IF(unpacked_tensor.size() < 4, - "The initializer [", name, "] should have one or more floats ", - "with size no less than 4, actual size: ", unpacked_tensor.size()); - scale = reinterpret_cast(unpacked_tensor.data())[0]; + scale = unpacked_tensor.DataAsSpan()[0]; } if (quant_param.zero_point) { // get the zero point if it's there - std::vector unpacked_tensor; const auto& name = quant_param.zero_point->Name(); - ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor)); - ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty"); + Initializer unpacked_tensor(*initializers.at(name), model_path); // Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI - zero_point = static_cast(unpacked_tensor[0]); + zero_point = static_cast(unpacked_tensor.DataAsByteSpan()[0]); } return Status::OK(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 40597c0cfe..6164bf9743 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -13,6 +13,7 @@ #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" +#include "core/optimizer/initializer.h" #include "helper.h" #include "op_builder.h" @@ -285,26 +286,15 @@ Status ModelBuilder::RegisterInitializers() { if (Contains(skipped_initializers_, tensor.name())) continue; - uint32_t index; - size_t size, padded_size; - std::tie(index, size, padded_size) = initializers[i++]; + auto [index, size, padded_size] = initializers[i++]; const uint8_t* src = nullptr; - // uint8_t data need unpack, need a holder for free memory after copy - std::vector unpacked_tensor; - switch (tensor.data_type()) { - case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: - case ONNX_NAMESPACE::TensorProto_DataType_UINT8: - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, graph_viewer_.ModelPath(), unpacked_tensor)); - ORT_RETURN_IF_NOT(size == unpacked_tensor.size(), - "initializer tensor: ", tensor.name(), "'s size: ", unpacked_tensor.size(), - " should match the calculated size: ", size); - src = unpacked_tensor.data(); - break; - // default: - // We should not get anything else here since we already checked in the 1st pass - } - + // TensorProto_DataType_UINT8 or TensorProto_DataType_FLOAT: + Initializer unpacked_tensor(tensor, graph_viewer_.ModelPath()); + size_t size_in_bytes = unpacked_tensor.DataAsByteSpan().size(); + ORT_RETURN_IF_NOT(size == size_in_bytes, + "initializer tensor: ", tensor.name(), "'s size: ", + size_in_bytes, " should match the calculated size: ", size); + src = unpacked_tensor.DataAsByteSpan().data(); uint8_t* dest = nnapi_model_->mem_initializers_->GetDataPtr() + offset; memcpy(dest, src, size); ORT_RETURN_IF_ERROR(SetOperandValue(index, nnapi_model_->mem_initializers_.get(), size, offset)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 0354b1aeb0..a3b0ea05c9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -4,6 +4,7 @@ #include "op_builder.h" #include +#include #include "core/common/logging/logging.h" #include "core/common/safeint.h" @@ -17,6 +18,7 @@ #include "helper.h" #include "model_builder.h" #include "op_support_checker.h" +#include "core/optimizer/initializer.h" using namespace android::nn::wrapper; @@ -126,9 +128,8 @@ static Status GetAxesForSqueezeAndUnSqueeze(ModelBuilder& model_builder, const N if (node_unit.Inputs().size() > 1) { const auto& initializers(model_builder.GetInitializerTensors()); const auto& axes_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(axes_tensor, unpacked_tensor)); - const int64_t* raw_axes = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(axes_tensor); + auto raw_axes = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(axes_tensor.dims()[0]); axes.resize(size); for (uint32_t i = 0; i < size; i++) { @@ -163,16 +164,11 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder, // TODO support other data types const uint8_t* src = nullptr; - std::vector unpacked_tensor; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: case ONNX_NAMESPACE::TensorProto_DataType_UINT8: case ONNX_NAMESPACE::TensorProto_DataType_INT8: { - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(), - unpacked_tensor)); - src = unpacked_tensor.data(); break; } default: @@ -180,7 +176,8 @@ static Status AddInitializerInNewLayout(ModelBuilder& model_builder, "The initializer of graph ", name, " doesn't have valid type: ", tensor.data_type()); } - + Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath()); + src = unpacked_tensor.DataAsByteSpan().data(); const auto out_t = shape[0], in_t = shape[1], h_t = shape[2], w_t = shape[3]; Shape dest_shape; @@ -248,15 +245,10 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder, // TODO support other data types const uint8_t* src = nullptr; - std::vector unpacked_tensor; switch (tensor.data_type()) { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: case ONNX_NAMESPACE::TensorProto_DataType_UINT8: case ONNX_NAMESPACE::TensorProto_DataType_INT8: { - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(), - unpacked_tensor)); - src = unpacked_tensor.data(); break; } default: @@ -264,7 +256,9 @@ static Status AddInitializerTransposed(ModelBuilder& model_builder, "The initializer of graph ", name, " doesn't have valid type: ", tensor.data_type()); } - + Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath()); + // could be float/u8/s8, so we have to use raw data here. + src = unpacked_tensor.DataAsByteSpan().data(); const auto x_t = shape[0], y_t = shape[1]; Shape dest_shape = {y_t, x_t}; OperandType operand_type = source_operand_type; @@ -422,11 +416,10 @@ static Status GetConvMatMulOpQuantizationScaleAndZeroPoint( w_zero_point = 0; // We need to copy the 1d scales array for per-channel quantization - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_tensor)); - const float* scales = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(scale_tensor); + auto scales = unpacked_tensor.DataAsSpan(); const size_t scales_size = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; - std::vector scales_vec(scales, scales + scales_size); + std::vector scales_vec(scales.begin(), scales.begin() + scales_size); w_scales = onnxruntime::make_optional(std::move(scales_vec)); return Status::OK(); } @@ -892,9 +885,8 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons auto input = node_unit.Inputs()[0].node_arg.Name(); const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(shape_tensor, unpacked_tensor)); - const int64_t* raw_shape = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(shape_tensor); + auto raw_shape = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(shape_tensor.dims()[0]); Shape input_shape = shaper[input]; @@ -1006,21 +998,15 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu a.reserve(size); b.reserve(size); - std::vector unpacked_scale_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scale_tensor, unpacked_scale_tensor)); - const float* scale_data = reinterpret_cast(unpacked_scale_tensor.data()); + Initializer unpacked_scale_tensor(scale_tensor); + Initializer unpacked_bias_tensor(bias_tensor); + Initializer unpacked_mean_tensor(mean_tensor); + Initializer unpacked_var_tensor(var_tensor); - std::vector unpacked_bias_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_bias_tensor)); - const float* bias_data = reinterpret_cast(unpacked_bias_tensor.data()); - - std::vector unpacked_mean_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(mean_tensor, unpacked_mean_tensor)); - const float* mean_data = reinterpret_cast(unpacked_mean_tensor.data()); - - std::vector unpacked_var_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(var_tensor, unpacked_var_tensor)); - const float* var_data = reinterpret_cast(unpacked_var_tensor.data()); + auto scale_data = unpacked_scale_tensor.DataAsSpan(); + auto bias_data = unpacked_bias_tensor.DataAsSpan(); + auto mean_data = unpacked_mean_tensor.DataAsSpan(); + auto var_data = unpacked_var_tensor.DataAsSpan(); for (int64_t i = 0; i < size; i++) { a.push_back(scale_data[i] / sqrt(var_data[i] + eps)); @@ -1390,12 +1376,10 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N Shape bias_dimen; for (auto dim : bias_tensor.dims()) bias_dimen.push_back(SafeInt(dim)); - - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor)); + Initializer unpacked_tensor(bias_tensor); OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale); ORT_RETURN_IF_ERROR( - model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); + model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); } const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); @@ -1819,11 +1803,12 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N Shape bias_dimen; for (auto dim : bias_tensor.dims()) bias_dimen.push_back(SafeInt(dim)); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(bias_tensor, unpacked_tensor)); + Initializer unpacked_tensor(bias_tensor); OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, a_scale * b_scale); ORT_RETURN_IF_ERROR( - model_builder.AddOperandFromPersistMemoryBuffer(bias, unpacked_tensor.data(), bias_operand_type)); + model_builder.AddOperandFromPersistMemoryBuffer( + bias, + unpacked_tensor.data(), bias_operand_type)); bias_idx = operand_indices.at(bias); } @@ -2381,17 +2366,15 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const if (inputs.size() == 3) { // we are using scales const auto& scales_name = inputs[2].node_arg.Name(); const auto& scales_tensor = *initializers.at(scales_name); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor)); - const float* scales_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(scales_tensor); + auto scales_data = unpacked_tensor.DataAsSpan(); ORT_RETURN_IF_ERROR( shaper.ResizeUsingScales(input, scales_data[h_idx], scales_data[w_idx], use_nchw, output)); } else { // we are using sizes const auto& sizes_name = inputs[3].node_arg.Name(); const auto& sizes_tensor = *initializers.at(sizes_name); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor)); - const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(sizes_tensor); + auto sizes_data = unpacked_tensor.DataAsSpan(); ORT_RETURN_IF_ERROR( shaper.ResizeUsingOutputSizes(input, SafeInt(sizes_data[h_idx]), SafeInt(sizes_data[w_idx]), use_nchw, output)); } @@ -2502,33 +2485,25 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const indices_data_type != ONNX_NAMESPACE::TensorProto_DataType_INT32) { // Add indices operand into nnapi const auto& indices_tensor = *initializers.at(input2); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(indices_tensor, unpacked_tensor)); + Initializer unpacked_tensor(indices_tensor); const auto data_type = indices_tensor.data_type(); const auto indices_shape = indices_tensor.dims(); - uint32_t size = 1; Shape indices_dimen; indices_dimen.reserve(indices_tensor.dims_size()); for (auto i = 0; i < indices_tensor.dims_size(); i++) { - size *= SafeInt(indices_shape[i]); indices_dimen.push_back(static_cast(indices_shape[i])); } - std::vector indices(size); - // see https://gist.github.com/shafik/848ae25ee209f698763cffee272a58f8#type-punning-arrays for the usage of memcpy here + std::vector indices(unpacked_tensor.size()); + if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT64) { - for (uint32_t i = 0; i < size; i++) { - int64_t index_i64; - memcpy(&index_i64, unpacked_tensor.data() + i * sizeof(int64_t), sizeof(int64_t)); - indices[i] = SafeInt(index_i64); - } + auto indice_span = unpacked_tensor.DataAsSpan(); + std::transform(indice_span.begin(), indice_span.end(), indices.begin(), + [](int64_t indice_n) -> int32_t { return SafeInt(indice_n); }); } else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32) { - for (uint32_t i = 0; i < size; i++) { - int32_t index; - memcpy(&index, unpacked_tensor.data() + i * sizeof(int32_t), sizeof(int32_t)); - indices[i] = SafeInt(index); - } + auto indice_span = unpacked_tensor.DataAsSpan(); + indices.assign(indice_span.begin(), indice_span.end()); } OperandType indices_operand_type(Type::TENSOR_INT32, indices_dimen); @@ -2684,20 +2659,14 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& initializers(model_builder.GetInitializerTensors()); const auto& tensor = *initializers.at(input_name); - std::vector unpacked_tensor; - ORT_RETURN_IF_ERROR( - onnxruntime::utils::UnpackInitializerData(tensor, model_builder.GetGraphViewer().ModelPath(), - unpacked_tensor)); - size_t tensor_byte_size = unpacked_tensor.size(); + Initializer unpacked_tensor(tensor, model_builder.GetGraphViewer().ModelPath()); const auto data_type = tensor.data_type(); if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT64) { - const int64_t* tensor_data = reinterpret_cast(unpacked_tensor.data()); - size_t size = tensor_byte_size / sizeof(int64_t); - data.insert(data.end(), tensor_data, tensor_data + size); + auto tensor_data = unpacked_tensor.DataAsSpan(); + data.insert(data.end(), tensor_data.begin(), tensor_data.end()); } else if (data_type == ONNX_NAMESPACE::TensorProto_DataType_INT32) { - const int32_t* tensor_data = reinterpret_cast(unpacked_tensor.data()); - size_t size = tensor_byte_size / sizeof(int32_t); - data.insert(data.end(), tensor_data, tensor_data + size); + auto tensor_data = unpacked_tensor.DataAsSpan(); + data.insert(data.end(), tensor_data.begin(), tensor_data.end()); } else { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Data type for starts and ends inputs' is not supported in this build. Got ", @@ -2843,29 +2812,14 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto* pads_initializer = model_builder.GetConstantInitializer(pads); ORT_RETURN_IF_NOT(pads_initializer, "pads must be a constant"); - std::vector pads_initializer_raw_data{}; - ORT_RETURN_IF_ERROR(utils::UnpackInitializerData(*pads_initializer, pads_initializer_raw_data)); + Initializer pads_initializer_raw_data(*pads_initializer); // assume pads_initializer has int64 data, per ONNX spec - ORT_RETURN_IF_NOT(pads_initializer_raw_data.size() == 2 * data_rank * sizeof(int64_t), - "Expected pads initializer size in bytes: ", 2 * data_rank * sizeof(int64_t), - ", actual: ", pads_initializer_raw_data.size()); - std::vector converted_pads_data{}; converted_pads_data.reserve(2 * data_rank); - - auto copy_and_convert = [](const void* raw_i64_src, - std::back_insert_iterator i32_dst) { - int64_t i64; - memcpy(&i64, raw_i64_src, sizeof(i64)); - *i32_dst = SafeInt(i64); - }; - + auto pads_span = pads_initializer_raw_data.DataAsSpan(); for (size_t i = 0; i < data_rank; ++i) { - copy_and_convert(&pads_initializer_raw_data[i * sizeof(int64_t)], - std::back_inserter(converted_pads_data)); - - copy_and_convert(&pads_initializer_raw_data[(i + data_rank) * sizeof(int64_t)], - std::back_inserter(converted_pads_data)); + converted_pads_data.push_back(SafeInt(pads_span[i])); + converted_pads_data.push_back(SafeInt(pads_span[i + data_rank])); } const Shape converted_pads_shape{data_rank, 2}; @@ -2880,15 +2834,8 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto& constant_value = inputs[2].node_arg.Name(); const auto* constant_value_initializer = model_builder.GetConstantInitializer(constant_value); ORT_RETURN_IF_NOT(constant_value_initializer, "constant_value must be a constant"); - - std::vector pad_value_raw_data{}; - ORT_RETURN_IF_ERROR(utils::UnpackInitializerData(*constant_value_initializer, pad_value_raw_data)); - // assume constant_value_initializer has float data - // ONNX spec says it matches `data` input type, and op support checker limits that to float - ORT_RETURN_IF_NOT(pad_value_raw_data.size() == sizeof(float), - "Expected constant_value initializer size in bytes: ", sizeof(float), - ", actual size: ", pad_value_raw_data.size()); - memcpy(&pad_value, pad_value_raw_data.data(), sizeof(float)); + Initializer pad_value_raw_data_init(*constant_value_initializer); + pad_value = pad_value_raw_data_init.DataAsSpan()[0]; } ADD_SCALAR_OPERAND(model_builder, input_indices, pad_value); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index f5a033766a..aebba40590 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -12,7 +12,7 @@ #include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" #include "core/providers/shared/node_unit/node_unit.h" #include "core/providers/shared/utils/utils.h" - +#include "core/optimizer/initializer.h" namespace onnxruntime { namespace nnapi { @@ -184,18 +184,10 @@ static bool IsQuantizationZeroPointSupported(const InitializedTensorSet& initial << " zero point dimension " << zero_dim; return false; } - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, model_path, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name - << ", error msg: " << status.ErrorMessage(); - return false; - } - + Initializer unpacked_tensor(zero_tensor, model_path); // Verify all onnx weight zero point(s) are 0(s) - const int8_t* zero_points = reinterpret_cast(unpacked_tensor.data()); - for (size_t i = 0; i < unpacked_tensor.size(); i++) { + auto zero_points = unpacked_tensor.DataAsSpan(); + for (int64_t i = 0; i < unpacked_tensor.size(); i++) { if (zero_points[i] != 0) { LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only support 0 as zero point, " << "zero_points[" << i << "] has value: " << zero_points[i]; @@ -346,7 +338,7 @@ class BaseOpSupportChecker : public IOpSupportChecker { const OpSupportCheckParams& params) const; virtual int GetMinSupportedOpSet(const NodeUnit& /* node_unit */) const { return 1; } - virtual int GetMaxSupportedOpSet(const NodeUnit& /* node_unit */) const { return 15; } + virtual int GetMaxSupportedOpSet(const NodeUnit& /* node_unit */) const { return 17; } // Check if this node_unit's type is supported // SingleNode type NodeUnit is supported @@ -731,13 +723,8 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init } const auto& perm_tensor = *initializers.at(perm_name); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(perm_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking perm_tensor: " << status.ErrorMessage(); - return false; - } - const int64_t* raw_perm = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(perm_tensor); + auto raw_perm = unpacked_tensor.DataAsSpan(); const auto perm_size = SafeInt(perm_tensor.dims()[0]); NodeAttrHelper helper(node_unit); @@ -1996,13 +1983,8 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi // We want to check if the scales or sizes are not trying to resize on N/C channels here if (inputs.size() == 3) { // we are using scales const auto& scales_tensor = *initializers.at(inputs[2].node_arg.Name()); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking scales_tensor: " << status.ErrorMessage(); - return false; - } - const float* scales_data = reinterpret_cast(unpacked_tensor.data()); + Initializer unpacked_tensor(scales_tensor); + auto scales_data = unpacked_tensor.DataAsSpan(); float scale_n = scales_data[0]; float scale_c = IsNodeLayoutNHWC(node_unit) ? scales_data[3] : scales_data[1]; if (scale_n != 1.0f || scale_c != 1.0f) { @@ -2015,15 +1997,9 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi // we are using sizes const auto& sizes_name = inputs[3].node_arg.Name(); const auto& sizes_tensor = *initializers.at(sizes_name); - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking sizes_tensor: " << status.ErrorMessage(); - return false; - } - + Initializer unpacked_tensor(sizes_tensor); int channel_idx = IsNodeLayoutNHWC(node_unit) ? 3 : 1; - const int64_t* sizes_data = reinterpret_cast(unpacked_tensor.data()); + auto sizes_data = unpacked_tensor.DataAsSpan(); uint32_t size_n = SafeInt(sizes_data[0]); uint32_t size_c = SafeInt(sizes_data[channel_idx]); if (size_n != input_shape[0] || size_c != input_shape[channel_idx]) { @@ -2352,20 +2328,12 @@ bool PadOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initiali } const ONNX_NAMESPACE::TensorProto& pads_initializer = *pads_initializer_it->second; - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(pads_initializer, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Error while unpacking pads initializer: " << status.ErrorMessage(); - return false; - } - - int64_t pad_value; - ORT_ENFORCE(unpacked_tensor.size() % sizeof(pad_value) == 0); - for (size_t i = 0; i < unpacked_tensor.size(); i += sizeof(pad_value)) { - memcpy(&pad_value, &unpacked_tensor[i], sizeof(pad_value)); - if (pad_value < 0) { + Initializer unpacked_tensor(pads_initializer); + auto tensor_data = unpacked_tensor.DataAsSpan(); + for (int64_t i = 0; i < unpacked_tensor.size(); i++) { + if (tensor_data[i] < 0) { LOGS_DEFAULT(VERBOSE) << "Negative pad value is not supported: pads[" - << i / sizeof(pad_value) << "] = " << pad_value; + << i << "] = " << tensor_data[i]; return false; } } diff --git a/onnxruntime/core/providers/nuphar/common/analysis/analysis.h b/onnxruntime/core/providers/nuphar/common/analysis/analysis.h deleted file mode 100644 index 9c4e771761..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/analysis.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/common/common.h" -#include "core/common/common.h" -#include "core/graph/graph_viewer.h" -#include "core/providers/nuphar/common/nuphar_subgraph.h" - -namespace onnxruntime { -namespace nuphar { - -// abstract class for Analysis -template -class AnalysisBase { - public: - AnalysisBase() {} - - AnalysisBase(const std::string& name) - : name_(name) {} - - virtual ~AnalysisBase() = default; - - virtual void Evaluate(INPUT_TYPE) = 0; - - const std::string& Name() const { - return name_; - } - - protected: - const std::string name_{"Unknown"}; - - private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(AnalysisBase); -}; - -using OrtAnalysis = AnalysisBase; -using NupharAnalysis = AnalysisBase; - -// Add Promote for OrtAnalysis and NupharAnalysis -DYNAMIC_PROMOTE(OrtAnalysis) -DYNAMIC_PROMOTE(NupharAnalysis) - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/graph_stats.h b/onnxruntime/core/providers/nuphar/common/analysis/graph_stats.h deleted file mode 100644 index f86a28695b..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/graph_stats.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/codegen/common/common.h" -#include "core/common/common.h" -#include "core/graph/graph_viewer.h" -#include "core/providers/nuphar/common/analysis/analysis.h" - -#include "core/providers/nuphar/common/nuphar_subgraph.h" -// Base class of GraphStatsBase -// GraphStatsBase holds analysis results from a graph -// GraphStatsBase can hold multiple analyses - -namespace onnxruntime { -namespace nuphar { - -template -class GraphStatsBase { - public: - GraphStatsBase(const std::string& name) - : name_(name) {} - - GraphStatsBase() {} - - virtual ~GraphStatsBase() = default; - - // Evaluate all passes - virtual void Evaluate(INPUT_TYPE graph) { - for (auto& pass : passes_) { - pass->Evaluate(graph); - } - } - - // Set passes externally - void SetAllPasses(const std::vector>>& passes) { - passes_.clear(); - for (auto& pass : passes) { - passes_.push_back(pass); - } - } - - // Set existed evaluated passes externally - void SetAllExistedEvaluatedPasses( - const std::vector>>& passes) { - existed_eval_passes_.clear(); - for (auto& pass : passes) { - existed_eval_passes_.push_back(pass); - } - } - - const std::string& Name() const { - return name_; - } - - protected: - const std::string name_{"Unknown"}; - - std::vector>> passes_; - - private: - // existed eval passes not requiring evaluation - std::vector>> existed_eval_passes_; - - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(GraphStatsBase); -}; - -using OrtGraphStats = GraphStatsBase; -using NupharSubgraphUnitStats = GraphStatsBase; - -// Add Promote for OrtGraphStats and NupharSubgraphUnitStats -DYNAMIC_PROMOTE(OrtGraphStats) -DYNAMIC_PROMOTE(NupharSubgraphUnitStats) - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/output_alias_analysis.cc b/onnxruntime/core/providers/nuphar/common/analysis/output_alias_analysis.cc deleted file mode 100644 index 06c33190a0..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/output_alias_analysis.cc +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/analysis/output_alias_analysis.h" - -#include "core/codegen/common/common.h" - -namespace onnxruntime { -namespace nuphar { - -void OutputAliasAnalysis::Traverse(gsl::span nodes, - const InlinedHashSet& graph_inputs, - const InlinedHashSet& graph_outputs) { - for (auto& node : nodes) { - if (node->NodeType() == Node::Type::Fused) { - // unboxing of fused node - const auto& func_body = GraphViewer(node->GetFunctionBody()->Body()); - Traverse(ConvertGraphNodesToNodePtrs(func_body.Nodes()), graph_inputs, graph_outputs); - } else { - // TODO: change identity to other alias - bool is_identity = (node->OpType() == "Identity"); - ORT_THROW_IF_ERROR(node->ForEachWithIndex( - node->OutputDefs(), - [&](const NodeArg& def, size_t) { - if (graph_outputs.count(def.Name()) > 0) { - NodeKey key = GetKey(node); - output_nodes_.insert(key); - if (is_identity) { - auto input_def = node->InputDefs()[0]; - // regard as aliased if input_def is not graph input - // otherwise, we still generate Identity ops in TVM - // TODO: remove once we have a better solution for alias optimization - if (graph_inputs.count(input_def->Name()) == 0) { - alias_use_defs_.insert(std::make_pair(key, input_def)); - NodeKey input_key = GetKey(input_def); - output_nodes_.insert(input_key); - } - } - } - return Status::OK(); - })); - } - } -} - -// TODO: please reimplement output alias using the right algorithm. -// Currently we only copy it from old graph_stats, which is still wrong one -void OutputAliasAnalysis::Evaluate(const onnxruntime::nuphar::NupharSubgraphUnit& graph) { - if (graph.IsSingleNode()) { - const Node* node = graph.nodes.front(); - auto subgraph = GetSubgraph(*node); - - if (nullptr != subgraph) { - const auto& graph_viewer = GraphViewer(*subgraph); - InlinedHashSet graph_inputs; - InlinedHashSet graph_outputs; - graph_inputs.reserve(graph_viewer.GetInputs().size()); - graph_outputs.reserve(graph_viewer.GetOutputs().size()); - for (const auto* def : graph_viewer.GetInputs()) { - if (nullptr != def) { - graph_inputs.insert(def->Name()); - } - } - for (const auto* def : graph_viewer.GetOutputs()) { - if (nullptr != def) { - graph_outputs.insert(def->Name()); - } - } - Traverse(ConvertGraphNodesToNodePtrs(graph_viewer.Nodes()), graph_inputs, graph_outputs); - } else { - NodeKey key = GetKey(node); - output_nodes_.insert(key); - } - } else { - // outputs names - InlinedHashSet graph_inputs; - InlinedHashSet graph_outputs; - graph_inputs.reserve(graph.inputs.size()); - graph_outputs.reserve(graph.outputs.size()); - for (const auto* def : graph.inputs) { - if (nullptr != def) { - graph_inputs.insert(def->Name()); - } - } - for (const auto* def : graph.outputs) { - if (nullptr != def) { - graph_outputs.insert(def->Name()); - } - } - Traverse(graph.nodes, graph_inputs, graph_outputs); - } -} - -bool OutputAliasAnalysis::IsOutputNode(const onnxruntime::Node* node) const { - return output_nodes_.count(GetKey(node)) != 0; -} - -bool OutputAliasAnalysis::IsOutputAlias(const onnxruntime::Node* node) const { - auto key = GetKey(node); - return alias_use_defs_.count(key) != 0; -} - -const onnxruntime::NodeArg* -OutputAliasAnalysis::SourceDefOfOutputAlias(const onnxruntime::NodeArg* node) const { - auto iter = alias_use_defs_.find(GetKey(node)); - if (iter != alias_use_defs_.end()) { - return iter->second; - } - return nullptr; -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/output_alias_analysis.h b/onnxruntime/core/providers/nuphar/common/analysis/output_alias_analysis.h deleted file mode 100644 index 31c6170119..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/output_alias_analysis.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/common/inlined_containers.h" -#include "core/codegen/common/common.h" -#include "core/graph/graph.h" -#include "core/providers/nuphar/common/analysis/analysis.h" -#include - -namespace onnxruntime { -namespace nuphar { - -class OutputAliasAnalysis : public NupharAnalysis { - public: - OutputAliasAnalysis() - : NupharAnalysis("OutputAliasAnalysis") {} - - ~OutputAliasAnalysis() = default; - - void Evaluate(const onnxruntime::nuphar::NupharSubgraphUnit& graph) override; - - bool IsOutputNode(const onnxruntime::Node* node) const; - - bool IsOutputAlias(const onnxruntime::Node* node) const; - - const onnxruntime::NodeArg* SourceDefOfOutputAlias(const onnxruntime::NodeArg* node) const; - - private: - // a set for output nodes - std::set output_nodes_; - // a map from an output alias to its input - std::map alias_use_defs_; - - void Traverse(gsl::span nodes, - const InlinedHashSet& graph_inputs, - const InlinedHashSet& graph_outputs); - - private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OutputAliasAnalysis); -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/shape_expr.h b/onnxruntime/core/providers/nuphar/common/analysis/shape_expr.h deleted file mode 100644 index 76495c77df..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/shape_expr.h +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/common/common.h" - -// TODO retire this file - -namespace onnxruntime { - -// A mini IR layer for shape inference -// Currently just use tvm::Expr but can be replaced by others later -// Following features are needed: -// 1. represent symbolic int -// 2. represent +-*/ -// 3. check if two DimExpr is the same -// 4. simplify if needed -// For now only symbolic int is supported -class SimpleDimExpr { - public: - SimpleDimExpr() : has_value_(false) {} - SimpleDimExpr(int64_t i) : value_(i), has_value_(true) {} - SimpleDimExpr(const std::string& sym) : symbol_(sym), has_value_(false) {} - bool IsConst() const { return has_value_; } - bool IsOne() const { return has_value_ && value_ == 1; } - bool operator==(const SimpleDimExpr& expr) const { - if (has_value_ != expr.has_value_) - return false; - - if (has_value_) - return value_ == expr.value_; - else - return symbol_ == expr.symbol_; - } - - bool operator!=(const SimpleDimExpr& expr) const { - return !(*this == expr); - } - - SimpleDimExpr operator+(const SimpleDimExpr& other) const { - ORT_ENFORCE(has_value_ && other.has_value_); - return SimpleDimExpr(value_ + other.value_); - } - - SimpleDimExpr operator-(const SimpleDimExpr& other) const { - ORT_ENFORCE(has_value_ && other.has_value_); - return SimpleDimExpr(value_ - other.value_); - } - - SimpleDimExpr operator*(const SimpleDimExpr& other) const { - if (has_value_ && other.has_value_) - return SimpleDimExpr(value_ * other.value_); - else if (IsOne()) - return other; - else if (other.IsOne()) - return *this; - else - ORT_ENFORCE(false, "unsupported symbolic dim computation"); - } - - SimpleDimExpr operator/(const SimpleDimExpr& other) const { - if (has_value_ && other.has_value_) - return SimpleDimExpr(value_ / other.value_); - else if (other.IsOne()) - return *this; - else - ORT_ENFORCE(false, "unsupported symbolic dim computation"); - } - - SimpleDimExpr operator%(const SimpleDimExpr& other) const { - ORT_ENFORCE(has_value_ && other.has_value_); - return SimpleDimExpr(value_ % other.value_); - } - - int64_t Value() const { - ORT_ENFORCE(IsConst()); - return value_; - } - - const std::string& Symbol() const { - ORT_ENFORCE(!IsConst()); - return symbol_; - } - - std::string ToString() const { - if (has_value_) - return std::to_string(value_); - else - return symbol_; - } - - private: - std::string symbol_; - int64_t value_; - bool has_value_; -}; - -template -class ShapeExprT { - public: - ShapeExprT() = default; - ShapeExprT(const ShapeExprT& expr) = default; - ShapeExprT(ShapeExprT&& expr) = default; - ShapeExprT(size_t size) { dims_.resize(size); } - ShapeExprT(const std::vector& dims) : dims_(dims) {} - ShapeExprT(const std::vector& dims) { - for (auto dim : dims) - dims_.push_back(DimT(dim)); - } - - size_t Rank() const { - return dims_.size(); - } - - int64_t TotalKnown() const { - if (dims_.size() == 0) - return 1; - int64_t total = 1; - for (size_t i = 0; i < dims_.size(); ++i) { - if (dims_[i].IsConst()) - total = total * dims_[i].Value(); - } - return total; - } - - size_t KnownFromDimension() const { - size_t min_index = dims_.size(); - for (int i = static_cast(dims_.size() - 1); i >= 0; i--) { - if (!dims_[i].IsConst()) - break; - min_index = static_cast(i); - } - return min_index; - } - - std::vector TailedKnown() const { - std::vector result; - - for (size_t i = KnownFromDimension(); i < Rank(); ++i) { - result.push_back(dims_[i].Value()); - } - return result; - } - - int64_t TotalTailedKnown() const { - int64_t result = 1; - for (size_t i = KnownFromDimension(); i < Rank(); ++i) { - result *= dims_[i].Value(); - } - return result; - } - - /** - Return the total number of elements up to the specified dimension. - @param dim Return size up to this dimension. Value must be >= 0 and < this.Size(). - */ - DimT SizeToDimension(size_t dim) const { - DimT total(1); - for (size_t i = 0; i < std::min(dim, dims_.size()); ++i) - total = total * dims_[i]; - return total; - } - - /** - Return the total number of elements from the specified dimension to the end of the tensor shape. - @param dim Return size up to this dimension. 0 <= dimension < this.Size(). - */ - DimT SizeFromDimension(size_t dim) const { - DimT total(1); - for (size_t i = dim; i < dims_.size(); ++i) - total = total * dims_[i]; - return total; - } - - bool IsConst() const { - return std::all_of(dims_.begin(), dims_.end(), [](const DimT& dim) { return dim.IsConst(); }); - } - - bool operator==(const ShapeExprT& shape) const { - if (Rank() != shape.Rank()) - return false; - - for (size_t dim = 0; dim < Rank(); ++dim) { - if (dims_[dim] != (shape.dims_[dim])) - return false; - } - return true; - } - - const ShapeExprT& operator=(const ShapeExprT& shape) { - dims_ = shape.dims_; - return *this; - } - - const DimT& at(size_t dim) const { - ORT_ENFORCE(dim < Rank()); - return dims_[dim]; - } - - DimT& at(size_t dim) { - ORT_ENFORCE(dim < Rank()); - return dims_[dim]; - } - - const DimT& operator[](size_t dim) const { - ORT_ENFORCE(dim < Rank()); - return dims_[dim]; - } - - DimT& operator[](size_t dim) { - ORT_ENFORCE(dim < Rank()); - return dims_[dim]; - } - - const std::vector Value() const { - ORT_ENFORCE(IsConst()); - std::vector result; - for (size_t i = 0; i < Rank(); ++i) { - result.push_back(dims_[i].Value()); - } - return result; - } - - std::string ToString() const { - std::ostringstream oss; - oss << "("; - for (size_t i = 0; i < Rank(); ++i) { - if (i > 0) - oss << ", "; - oss << dims_[i].ToString(); - } - oss << ")"; - return oss.str(); - } - - private: - std::vector dims_; -}; - -typedef SimpleDimExpr DimExpr; -typedef ShapeExprT ShapeExpr; - -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_codegen_stats.cc b/onnxruntime/core/providers/nuphar/common/analysis/subgraph_codegen_stats.cc deleted file mode 100644 index d95d777479..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_codegen_stats.cc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" - -#include "core/providers/nuphar/common/analysis/output_alias_analysis.h" -#include "core/providers/nuphar/common/analysis/use_count_analysis.h" - -namespace onnxruntime { -namespace nuphar { - -// CodeGenUnitStats has two analysis passes -// The first pass, offset as 0, is UseCountAnalysis -// The second pass, offset as 1, is OutputAliasAnalysis -constexpr int UseCountAnalysisOffset = 0; -constexpr int OutputAliasAnalysisOffset = 1; - -// True reuse count for cheap Op -constexpr int CheapNodeTrueReuseCount = 2; - -// Constructor -CodeGenUnitStats::CodeGenUnitStats( - const std::shared_ptr& shape_infernece) - : NupharSubgraphUnitStats("CodeGenUnitStats") { - auto use_count_pass = std::make_shared(shape_infernece); - passes_.push_back(use_count_pass); - - auto output_alias_pass = std::make_shared(); - passes_.push_back(output_alias_pass); -} - -int CodeGenUnitStats::NodeUseCount(const onnxruntime::Node* node) const { - ORT_ENFORCE(passes_.size() > UseCountAnalysisOffset); - return Promote(passes_[UseCountAnalysisOffset])->NodeUseCount(node); -} - -bool CodeGenUnitStats::IsCheapNodeReuse(const onnxruntime::Node* node) const { - ORT_ENFORCE(passes_.size() > UseCountAnalysisOffset); - - // always inline(not reuse) alias node - if (IsAliasNode(*node)) - return false; - - // always inline tensor ops that do not involve computation - static const std::unordered_set trivial_tensor_ops = { - "Cast", - "Concat", - "Crop", - "Expand", - "Gather", - "GatherElements", - "Pad", - "Shape", - "Slice", - "Split", - "Tile", - "Transpose", - }; - if (trivial_tensor_ops.count(node->OpType()) > 0) - return false; - - // always inline cheap compute ops if reuse count is low - static const std::unordered_set trivial_compute_ops = { - "Abs", - "Add", - "Ceil", - "Clip", - "Equal", - "Floor", - "Greater", - "Less", - "Mul", - "Neg", - "Relu", - "Sub", - "Where", - }; - if (trivial_compute_ops.count(node->OpType()) > 0) - return Promote(passes_[UseCountAnalysisOffset])->NodeUseCount(node) > CheapNodeTrueReuseCount; - - // Otherwise return true and use count is determined by NodeUseCount - return true; -} - -bool CodeGenUnitStats::IsOutputNode(const onnxruntime::Node* node) const { - ORT_ENFORCE(passes_.size() > OutputAliasAnalysisOffset); - return Promote(passes_[OutputAliasAnalysisOffset])->IsOutputNode(node); -} - -bool CodeGenUnitStats::IsOutputAlias(const onnxruntime::Node* node) const { - ORT_ENFORCE(passes_.size() > OutputAliasAnalysisOffset); - return Promote(passes_[OutputAliasAnalysisOffset])->IsOutputAlias(node); -} - -const onnxruntime::NodeArg* CodeGenUnitStats::SourceDefOfOutputAlias(const onnxruntime::NodeArg* node) const { - ORT_ENFORCE(passes_.size() > OutputAliasAnalysisOffset); - return Promote(passes_[OutputAliasAnalysisOffset])->SourceDefOfOutputAlias(node); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_codegen_stats.h b/onnxruntime/core/providers/nuphar/common/analysis/subgraph_codegen_stats.h deleted file mode 100644 index 69b135f73e..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_codegen_stats.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/common/common.h" -#include "core/providers/nuphar/common/analysis/graph_stats.h" -#include "core/providers/nuphar/common/analysis/use_count_analysis.h" -#include "core/providers/nuphar/common/nuphar_subgraph.h" - -namespace onnxruntime { -namespace nuphar { - -class CodeGenUnitStats : public NupharSubgraphUnitStats { - public: - CodeGenUnitStats(const std::shared_ptr& shape_infernece); - - ~CodeGenUnitStats() = default; - - int NodeUseCount(const onnxruntime::Node* node) const; - - bool IsCheapNodeReuse(const onnxruntime::Node* node) const; - - bool IsOutputNode(const onnxruntime::Node* node) const; - - bool IsOutputAlias(const onnxruntime::Node* node) const; - - const onnxruntime::NodeArg* SourceDefOfOutputAlias(const onnxruntime::NodeArg* node) const; - - private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CodeGenUnitStats); -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_partition_stats.cc b/onnxruntime/core/providers/nuphar/common/analysis/subgraph_partition_stats.cc deleted file mode 100644 index b35b456bd6..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_partition_stats.cc +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/analysis/subgraph_partition_stats.h" - -#include "core/providers/nuphar/common/analysis/use_count_analysis.h" - -namespace onnxruntime { -namespace nuphar { - -// TODO: Add memory analysis -// SubgraphPartitionStats has one analysis pass -// The first pass, offset as 0, is UseCountAnalysis -constexpr int UseCountAnalysisOffset = 0; - -void SubgraphPartitionStats::SetShapeInference( - const std::shared_ptr& shape_infernece) { - passes_.clear(); - passes_.emplace_back(std::make_shared(shape_infernece)); -} - -int SubgraphPartitionStats::NodeUseCount(const onnxruntime::Node* node) const { - ORT_ENFORCE(passes_.size() > UseCountAnalysisOffset); - return Promote(passes_[UseCountAnalysisOffset])->NodeUseCount(node); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_partition_stats.h b/onnxruntime/core/providers/nuphar/common/analysis/subgraph_partition_stats.h deleted file mode 100644 index afbcf0d2a3..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/subgraph_partition_stats.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/codegen/common/common.h" -#include "core/providers/nuphar/common/analysis/graph_stats.h" -#include "core/providers/nuphar/compiler/traverse_shape_infer.h" - -namespace onnxruntime { -namespace nuphar { - -// TODO: rename class name to more target-specific in the tvm refactoring -// Maybe GraphPartitionStatsX86 -class SubgraphPartitionStats : public OrtGraphStats { - public: - SubgraphPartitionStats() - : OrtGraphStats("SubgraphPartitionStats") {} - - ~SubgraphPartitionStats() = default; - - void SetShapeInference(const std::shared_ptr& shape_infernece); - - int NodeUseCount(const onnxruntime::Node* node) const; - - private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SubgraphPartitionStats); -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/use_count_analysis.cc b/onnxruntime/core/providers/nuphar/common/analysis/use_count_analysis.cc deleted file mode 100644 index f774f79c76..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/use_count_analysis.cc +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/analysis/use_count_analysis.h" - -#include "core/codegen/common/common.h" -#include "core/graph/function.h" - -namespace onnxruntime { -namespace nuphar { - -constexpr int PRESET_USE_COUNT_FOR_UNKNOWN = 10; -constexpr int PRESET_USE_COUNT_FOR_SOFTMAX = 3; - -static void CountGemmOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts); - -static void CountMatMulOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts); - -static void CountRecurrentOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts); - -static void CountMatrixArgs(const onnxruntime::NodeArg* A, - const onnxruntime::NodeArg* B, - const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts); - -static void CountNodeArg(const onnxruntime::NodeArg* input_def, - const onnxruntime::Node& node, - gsl::span graph_inputs, - std::unordered_map& node_use_counts, - int use_cnt); - -static bool IsMatMulOp(const std::string& op) { - return op == "MatMul" || op == "MatMulInteger" || op == "MatMulInteger16"; -} - -void CountGemmOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts) { - ORT_ENFORCE(node.OpType() == "Gemm"); - - auto inputs = node.InputDefs(); - CountMatrixArgs(inputs[0], inputs[1], node, graph_inputs, shape_func, node_use_counts); - if (inputs.size() > 2) { - // C's use cnt is fixed. - CountNodeArg(inputs[2], node, graph_inputs, node_use_counts, 1); - } -} - -void CountMatMulOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts) { - ORT_ENFORCE(IsMatMulOp(node.OpType())); - auto inputs = node.InputDefs(); - CountMatrixArgs(inputs[0], inputs[1], node, graph_inputs, shape_func, node_use_counts); -} - -void CountRecurrentOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function, - std::unordered_map& node_use_counts) { - int use_count = PRESET_USE_COUNT_FOR_UNKNOWN; - - ORT_THROW_IF_ERROR(node.ForEachWithIndex( - node.InputDefs(), - [&node, &graph_inputs, &node_use_counts, &use_count](const NodeArg& def, size_t) { - CountNodeArg(&def, node, graph_inputs, node_use_counts, use_count); - return Status::OK(); - })); -} - -static bool IsSoftmaxOp(const std::string& op) { - return op == "Softmax" || op == "LogSoftmax"; -} - -void CountSoftmaxOp(const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function, - std::unordered_map& node_use_counts) { - // Use preset use count for Softmax/LogSoftmax input - int use_count = PRESET_USE_COUNT_FOR_SOFTMAX; - - ORT_THROW_IF_ERROR(node.ForEachWithIndex( - node.InputDefs(), - [&node, &graph_inputs, &node_use_counts, &use_count](const NodeArg& def, size_t) { - CountNodeArg(&def, node, graph_inputs, node_use_counts, use_count); - return Status::OK(); - })); -} - -void CountMatrixArgs(const onnxruntime::NodeArg* A, - const onnxruntime::NodeArg* B, - const onnxruntime::Node& node, - gsl::span graph_inputs, - std::function shape_func, - std::unordered_map& node_use_counts) { - int use_cnt = PRESET_USE_COUNT_FOR_UNKNOWN; - const ShapeExpr* a_shape = shape_func(A); - if (nullptr != a_shape) { - // B's use cnt is based on the rows of A - // skip symbolic dimensions for Sequence and batch - auto a_cols = (a_shape->Rank() > 0 && a_shape->at(a_shape->Rank() - 1).IsConst()) ? a_shape->at(a_shape->Rank() - 1).Value() : 1; - use_cnt = a_shape->TotalTailedKnown() / a_cols; - } - CountNodeArg(B, node, graph_inputs, node_use_counts, use_cnt); - - // reset use_cnt - use_cnt = PRESET_USE_COUNT_FOR_UNKNOWN; - const ShapeExpr* b_shape = shape_func(B); - if (nullptr != b_shape) { - const DimExpr& dim = b_shape->Rank() > 1 ? b_shape->at(b_shape->Rank() - 1) : DimExpr(1); - // A's use cnt is based on the cols of B. If B is 1-D, use cnt is 1 - if (dim.IsConst()) - use_cnt = dim.Value(); - } - - CountNodeArg(A, node, graph_inputs, node_use_counts, use_cnt); -} - -void CountNodeArg(const onnxruntime::NodeArg* input_def, - const onnxruntime::Node& node, - gsl::span graph_inputs, - std::unordered_map& node_use_counts, - int use_cnt) { - // Skip graph's input args nodes - if (std::find(graph_inputs.begin(), graph_inputs.end(), input_def) != graph_inputs.end()) - return; - - const Node* input_node = GetInputNode(node, input_def); - - if (nullptr != input_node) { - node_use_counts[GetKey(input_node)] += use_cnt; - } -} - -InternalUseCountAnalysis::InternalUseCountAnalysis(const std::shared_ptr& shape_inference) { - shape_func_ = [&shape_inference](const onnxruntime::NodeArg* X) { - return shape_inference->Lookup(X); - }; -} - -void InternalUseCountAnalysis::Traverse( - gsl::span nodes, - gsl::span graph_inputs, - gsl::span graph_outputs) { - for (auto& node : nodes) { - auto op_type = node->OpType(); - if (op_type == "Gemm") { - CountGemmOp(*node, graph_inputs, shape_func_, node_use_counts_); - } else if (IsMatMulOp(op_type)) { - CountMatMulOp(*node, graph_inputs, shape_func_, node_use_counts_); - } else if (op_type == "Scan") { - auto subgraph = node->GetGraphAttribute("body"); - Evaluate(GraphViewer(*subgraph)); - int use_count = PRESET_USE_COUNT_FOR_UNKNOWN; - ORT_THROW_IF_ERROR(node->ForEachWithIndex( - node->InputDefs(), - [this, &node, &graph_inputs, &use_count](const NodeArg& def, size_t) { - CountNodeArg(&def, *node, graph_inputs, node_use_counts_, use_count); - return Status::OK(); - })); - } else if (IsRecurrentNode(*node)) { - CountRecurrentOp(*node, graph_inputs, shape_func_, node_use_counts_); - } else if (node->NodeType() == Node::Type::Fused) { - // note: when unboxing subgraph in fused node, use outermost graph input/output - const auto& func_body = GraphViewer(node->GetFunctionBody()->Body()); - Traverse(ConvertGraphNodesToNodePtrs(func_body.Nodes()), graph_inputs, graph_outputs); - } else if (IsSoftmaxOp(op_type)) { - CountSoftmaxOp(*node, graph_inputs, shape_func_, node_use_counts_); - } else if (op_type != "Shape") { //don't count on Shape node input, because of no data dependency - int use_count = 1; - ORT_THROW_IF_ERROR(node->ForEachWithIndex( - node->InputDefs(), - [this, &node, &graph_inputs, &use_count](const NodeArg& def, size_t) { - CountNodeArg(&def, *node, graph_inputs, node_use_counts_, use_count); - return Status::OK(); - })); - } - - NodeKey key = GetKey(node); - // For any output_def of the node that is part of graph's outputs but not from graph.Nodes(), - // we need to increase the node's use cnt accordingly. Otherwise, we would lose those uses. - ORT_THROW_IF_ERROR(node->ForEachWithIndex( - node->OutputDefs(), - [this, &graph_outputs, &key](const NodeArg& def, size_t) { - if (std::find(graph_outputs.begin(), graph_outputs.end(), &def) != graph_outputs.end()) { - node_use_counts_[key]++; - } - return Status::OK(); - })); - } -} - -void InternalUseCountAnalysis::Evaluate(const onnxruntime::GraphViewer& graph) { - const auto& graph_inputs = graph.GetInputs(); - const auto& graph_outputs = graph.GetOutputs(); - Traverse(ConvertGraphNodesToNodePtrs(graph.Nodes()), graph_inputs, graph_outputs); -} - -void InternalUseCountAnalysis::Evaluate(const onnxruntime::nuphar::NupharSubgraphUnit& graph) { - const auto& graph_inputs = graph.inputs; - const auto& graph_outputs = graph.outputs; - Traverse(graph.nodes, graph_inputs, graph_outputs); -} - -void InternalUseCountAnalysis::IncrementCount(const onnxruntime::NodeArg* def) { - node_use_counts_[GetKey(def)]++; -} - -int InternalUseCountAnalysis::NodeUseCount(const onnxruntime::Node* node) const { - auto node_iter = node_use_counts_.find(GetKey(node)); - if (node_iter != node_use_counts_.end()) { - return node_iter->second; - } else { - return 0; - } -} - -OrtUseCountAnalysis::OrtUseCountAnalysis(const std::shared_ptr& shape_inference) - : OrtAnalysis("OrtUseCountAnalysis") { - internal_analysis_ = std::make_unique(shape_inference); -} - -void OrtUseCountAnalysis::Evaluate(const onnxruntime::GraphViewer& graph) { - internal_analysis_->Evaluate(graph); -} - -void OrtUseCountAnalysis::IncrementCount(const onnxruntime::NodeArg* def) { - internal_analysis_->IncrementCount(def); -} - -int OrtUseCountAnalysis::NodeUseCount(const onnxruntime::Node* node) const { - return internal_analysis_->NodeUseCount(node); -} - -NupharUseCountAnalysis::NupharUseCountAnalysis(const std::shared_ptr& shape_inference) - : NupharAnalysis("NupharUseCountAnalysis") { - internal_analysis_ = std::make_unique(shape_inference); -} - -void NupharUseCountAnalysis::Evaluate(const onnxruntime::nuphar::NupharSubgraphUnit& graph) { - internal_analysis_->Evaluate(graph); -} - -void NupharUseCountAnalysis::IncrementCount(const onnxruntime::NodeArg* def) { - internal_analysis_->IncrementCount(def); -} - -int NupharUseCountAnalysis::NodeUseCount(const onnxruntime::Node* node) const { - return internal_analysis_->NodeUseCount(node); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/analysis/use_count_analysis.h b/onnxruntime/core/providers/nuphar/common/analysis/use_count_analysis.h deleted file mode 100644 index 27267b3328..0000000000 --- a/onnxruntime/core/providers/nuphar/common/analysis/use_count_analysis.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/common/common.h" -#include "core/providers/nuphar/common/analysis/analysis.h" -#include "core/providers/nuphar/common/analysis/shape_expr.h" -#include "core/providers/nuphar/compiler/traverse_shape_infer.h" -#include "core/graph/graph.h" - -#include -#include - -// TODO change namespace from codegen to nuphar - -namespace onnxruntime { -namespace nuphar { - -class InternalUseCountAnalysis { - public: - InternalUseCountAnalysis(const std::shared_ptr& shape_inference); - - ~InternalUseCountAnalysis() = default; - - void Evaluate(const onnxruntime::GraphViewer& graph); - - void Evaluate(const NupharSubgraphUnit& graph); - - void IncrementCount(const onnxruntime::NodeArg* arg); - - int NodeUseCount(const onnxruntime::Node* node) const; - - private: - void Traverse(gsl::span nodes, - gsl::span graph_inputs, - gsl::span graph_outputs); - - std::unordered_map node_use_counts_; - std::function shape_func_; - - private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(InternalUseCountAnalysis); -}; - -// TODO analysis move to namespace nuphar - -class OrtUseCountAnalysis : public OrtAnalysis { - public: - OrtUseCountAnalysis(const std::shared_ptr& shape_inference); - ~OrtUseCountAnalysis() = default; - - void Evaluate(const onnxruntime::GraphViewer& graph) override; - - void IncrementCount(const onnxruntime::NodeArg* arg); - - int NodeUseCount(const onnxruntime::Node* node) const; - - private: - std::unique_ptr internal_analysis_; - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OrtUseCountAnalysis); -}; - -class NupharUseCountAnalysis : public NupharAnalysis { - public: - NupharUseCountAnalysis(const std::shared_ptr& shape_inference); - - ~NupharUseCountAnalysis() = default; - - void Evaluate(const onnxruntime::nuphar::NupharSubgraphUnit& graph) override; - - void IncrementCount(const onnxruntime::NodeArg* arg); - - int NodeUseCount(const onnxruntime::Node* node) const; - - private: - std::unique_ptr internal_analysis_; - - private: - ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(NupharUseCountAnalysis); -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc b/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc deleted file mode 100644 index af5812df8c..0000000000 --- a/onnxruntime/core/providers/nuphar/common/nuphar_settings.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/nuphar_settings.h" - -#include "core/codegen/common/common.h" -#include "core/codegen/common/utils.h" -#include "core/common/logging/logging.h" -#include "core/providers/nuphar/nuphar_execution_provider.h" - -#include -#include -#include -#include - -namespace onnxruntime { -namespace nuphar { - -static const std::unordered_set valid_keys = { - codegen::CodeGenSettings::kDumpAllOptions, - codegen::CodeGenSettings::kCodeGenDumpModule, - codegen::CodeGenSettings::kCodeGenDumpLower, - codegen::CodeGenSettings::kCodeGenDumpSchedule, - kNupharFastMath, - kNupharFastActivation, - kNupharForceNoTensorize, - kNupharTensorize_IGEMM_Tile_M, - kNupharTensorize_IGEMM_Tile_N, - kNupharTensorize_IGEMM_Tile_K, - kNupharTensorize_IGEMM_Permute, - kNupharTensorize_IGEMM_Split_Last_Tile, - kNupharDumpFusedNodes, - kNupharDumpPartition, - kNupharIMatMulForceMkl, - kNupharMatmulExec, - kNupharCachePath, - kNupharCacheSoName, - kNupharCacheModelChecksum, - kNupharCacheForceNoJIT, - kNupharCodeGenTarget, - kNupharParallelMinWorkloads}; - -void SetDefaultOptions(std::map& options) { - // create two temporary strings to get rid of the odr-use issue introduced - // The issue would trigger missing definition errors for static constexpr members - // at link time. - std::string fast_math_opt(kNupharFastMath); - std::string select_fast_math(kNupharFastMath_ShortPolynormial); - options.insert(std::make_pair(fast_math_opt, select_fast_math)); - - std::string fast_act_opt(kNupharFastActivation); - std::string select_fast_act(kNupharActivations_DeepCpu); - options.insert(std::make_pair(fast_act_opt, select_fast_act)); - - // set jit cache so name - std::string cache_so_name_opt(kNupharCacheSoName); - std::string cache_so_name_default(kNupharCacheSoName_Default); - options.insert(std::make_pair(cache_so_name_opt, cache_so_name_default)); - - std::string parallel_min_workloads_opt(kNupharParallelMinWorkloads); - // turn off parallel schedule by default to avoid TVM thread pool confliction with others - // this is to ensure performance when user runs multiple inference threads, with each runs as single thread - // if needed, user can override it with settings, and use TVM_NUM_THREADS to control the thread pool - unsigned int parallel_min_workloads_default = 0; - options.insert(std::make_pair(parallel_min_workloads_opt, std::to_string(parallel_min_workloads_default))); -} - -void CreateNupharCodeGenSettings(const NupharExecutionProviderInfo& info) { - std::map options; - SetDefaultOptions(options); - - std::unordered_set required_options; - if (!info.settings.empty()) { - const std::string& str = info.settings; - - // tokenize settings - std::regex reg("\\s*,\\s*"); - std::sregex_token_iterator iter(str.begin(), str.end(), reg, -1); - std::sregex_token_iterator iter_end; - std::vector pairs(iter, iter_end); - - ORT_ENFORCE(pairs.size() > 0); - for (const auto& pair : pairs) { - auto pos_colon = pair.find(':'); - ORT_ENFORCE(pos_colon != std::string::npos, "Invalid key value pair."); - std::string key = pair.substr(0, pos_colon); - std::string value = pair.substr(pos_colon + 1); - - // trim leading and trailing spaces from key/value - auto trim = [](const std::string& str) -> std::string { - const std::string WHITESPACE = " \n\r\t\f\v"; - size_t start = str.find_first_not_of(WHITESPACE); - if (start == std::string::npos) { - return ""; - } else { - size_t end = str.find_last_not_of(WHITESPACE); - ORT_ENFORCE(end != std::string::npos); - return str.substr(start, end + 1); - } - }; - key = trim(key); - value = trim(value); - - if (valid_keys.count(key) == 0) { - ORT_NOT_IMPLEMENTED("NupharCodeGenSettings: unknown option (", key, ")"); - } - required_options.insert(key); - options[key] = value; - } - } - -#ifndef GOLDEN_BUILD - // environment variables override existing settings - for (const auto& key : valid_keys) { - std::string env_key; - // env var is always upper case - std::transform(key.begin(), key.end(), std::back_inserter(env_key), (int (*)(int))std::toupper); - if (IsEnvVarDefined(env_key.c_str())) { - // value is case-sensitive - auto value = std::string(GetEnv(env_key.c_str()).get()); - - if (required_options.count(key) > 0 && options.at(key) != value) { - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) - << "NupharCodeGenSettings: option(" << key - << ") from environment variable is ignored because of existing required option value: " - << options.at(key); - } else { - options[key] = value; - } - } - } -#endif - - codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - settings.Clear(); // remove previous settings and start from scratch - - settings.InsertOptions(options); - - if (settings.HasOption(codegen::CodeGenSettings::kDumpAllOptions)) { - settings.DumpOptions(); - } -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_settings.h b/onnxruntime/core/providers/nuphar/common/nuphar_settings.h deleted file mode 100644 index 303daa98e3..0000000000 --- a/onnxruntime/core/providers/nuphar/common/nuphar_settings.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/common/settings.h" - -namespace onnxruntime { - -// forward declaration -struct NupharExecutionProviderInfo; - -namespace nuphar { -constexpr static const char* kNupharDumpPartition = "nuphar_dump_partition"; -constexpr static const char* kNupharDumpFusedNodes = "nuphar_dump_fused_nodes"; -constexpr static const char* kNupharMatmulExec = "nuphar_matmul_exec"; -constexpr static const char* kNupharCachePath = "nuphar_cache_path"; -constexpr static const char* kNupharCacheSoName = "nuphar_cache_so_name"; -constexpr static const char* kNupharCacheModelChecksum = "nuphar_cache_model_checksum"; -constexpr static const char* kNupharCacheForceNoJIT = "nuphar_cache_force_no_jit"; -// force to use IMatMulExternMKL/IMatMul16ExternMKL -constexpr static const char* kNupharIMatMulForceMkl = "nuphar_imatmul_force_mkl"; - -constexpr static const char* kNupharMatMulExec_ExternCpu = "extern_cpu"; - -constexpr static const char* kNupharForceNoTensorize = "nuphar_force_no_tensorize"; - -constexpr static const char* kNupharTensorize_IGEMM_Tile_M = "nuphar_igemm_tile_m"; -constexpr static const char* kNupharTensorize_IGEMM_Tile_N = "nuphar_igemm_tile_n"; -constexpr static const char* kNupharTensorize_IGEMM_Tile_K = "nuphar_igemm_tile_k"; - -constexpr static const char* kNupharTensorize_IGEMM_Permute = "nuphar_igemm_permute"; -constexpr static const char* kNupharTensorize_IGEMM_Permute_Inner = "inner"; -constexpr static const char* kNupharTensorize_IGEMM_Permute_Outer = "outer"; -constexpr static const char* kNupharTensorize_IGEMM_Permute_All = "all"; -constexpr static const char* kNupharTensorize_IGEMM_Split_Last_Tile = "nuphar_igemm_split_last_tile"; - -constexpr static const char* kNupharFastMath = "nuphar_fast_math"; // fast math -constexpr static const char* kNupharFastMath_Polynormial = "polynormial_math"; // generic polynormial fast math for exp and log -constexpr static const char* kNupharFastMath_ShortPolynormial = "short_polynormial_math"; // generic shorter polynormial fast math for exp and log - -constexpr static const char* kNupharFastActivation = "nuphar_fast_activation"; // fast activation -constexpr static const char* kNupharActivations_DeepCpu = "deep_cpu_activation"; - -// Option to control nuphar code generation target (avx / avx2 / avx512) -constexpr static const char* kNupharCodeGenTarget = "nuphar_codegen_target"; - -// Option to control nuphar code to run with parallel schedule -constexpr static const char* kNupharParallelMinWorkloads = "nuphar_parallel_min_workloads"; - -constexpr static const char* kNupharCacheSoName_Default = "jit.so"; - -void CreateNupharCodeGenSettings(const NupharExecutionProviderInfo& info); - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_subgraph.h b/onnxruntime/core/providers/nuphar/common/nuphar_subgraph.h deleted file mode 100644 index 52f950a648..0000000000 --- a/onnxruntime/core/providers/nuphar/common/nuphar_subgraph.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/framework/tensor.h" -#include "core/graph/graph.h" -#include "core/graph/graph_viewer.h" - -#include -#include -#include - -namespace onnxruntime { -namespace nuphar { - -using FindInitializerFunc = std::function; - -struct OrtSubgraphAllocationInfo { - std::unordered_map internal_allocator_offset; - std::unordered_map inputs; - std::unordered_map outputs; - int offset_count; - - OrtSubgraphAllocationInfo(const Node& node) : offset_count(0) { - int input_counter = 0; - int output_counter = 0; - - node.ForEachDef( - [&input_counter, &output_counter, this](const NodeArg& def, bool is_input) { - const std::string& def_name = def.Name(); - if (is_input) { - if (inputs.count(def_name) == 0) { - inputs.emplace(def_name, input_counter); - } - input_counter++; - } else { - outputs.emplace(def_name, output_counter++); - } - }); - } - - int CreateOrGetInternalAllocatorOffset(const std::string& def_name) { - if (internal_allocator_offset.count(def_name) > 0) { - return internal_allocator_offset.at(def_name); - } - internal_allocator_offset.insert(std::make_pair(def_name, offset_count)); - return offset_count++; - } -}; - -enum class NodeArgTileAttribute : int { - None = 0, - Forward = 1, - Backward = 2, - NoMerger = 3, -}; - -// NupharSubgraphUnit is a data struct under Ort Subgraph. -// It is a customized data struct in nuphar -// to enable concurrent function codegen within a Ort Kernel (which maps to an Ort Subgraph) -struct NupharSubgraphUnit { - NupharSubgraphUnit() { - id_ = counter++; - } - - std::vector nodes; - - // inputs include each input of this NupharSubgraphUnit (input of Partition AND this NupharSubgraphUnit at the same time) - // it also includes initializers - std::vector inputs; - - // outputs include each output of this NupharSubgraphUnit and real_output (output of Partition AND this NupharSubgraphUnit at the same time) - std::vector outputs; - - // initializers include each intializer of this NupharSubgraphUnit - std::map initializers; - - // optional - std::vector input_attrs; - std::vector output_attrs; - - bool IsSingleNode() const { - return nodes.size() == 1; - } - - std::string UniqueId() const { - return std::to_string(id_); - } - - public: - // counter used for subgraph id - // reset outside after cache generated - // to avoid same inference session continue - // increase the counter - thread_local static int64_t counter; - - private: - int64_t id_; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc b/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc deleted file mode 100644 index 78e6a260fe..0000000000 --- a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.cc +++ /dev/null @@ -1,279 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/nuphar_tvm_utils.h" - -#include "core/providers/nuphar/common/nuphar_subgraph.h" -#include "core/providers/nuphar/common/nuphar_settings.h" -#include "core/codegen/common/common.h" -#include "core/codegen/common/target_info.h" - -#include "core/common/logging/logging.h" -#include "core/platform/env.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/scripts/NUPHAR_CACHE_VERSION" -#include "gsl/gsl" -#include -#include -#define _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING // required by VS 2019 -#include -#undef _SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING -#include -#include -namespace fs = std::experimental::filesystem; - -namespace onnxruntime { -namespace nuphar { - -static bool GetOrCreateTVMModuleCacheDirectory(fs::path& path, bool create) { - codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - - if (!settings.HasOption(kNupharCachePath)) - return false; - - path = settings.GetOptionValue(kNupharCachePath); - if (!create && !fs::is_directory(path)) - return false; - - if (!fs::is_directory(path)) - if (!fs::create_directory(path)) { - throw std::runtime_error("Failed to create directory " + path.string()); - } - - path.append(__NUPHAR_CACHE_VERSION__); - if (!create && !fs::is_directory(path)) - return false; - - if (!fs::is_directory(path)) - if (!fs::create_directory(path)) { - throw std::runtime_error("Failed to create directory " + path.string()); - } - - return true; -} - -static bool GetCacheSoFilePath(std::string& so_path) { - codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - fs::path path; - if (!GetOrCreateTVMModuleCacheDirectory(path, /*create*/ false)) - return false; - - auto so_name = settings.GetOptionValue(kNupharCacheSoName); - path.append(so_name); - if (fs::is_regular_file(path)) { - so_path = path.string(); - return true; - } - return false; -} - -static void* GetFuncFromLibrary(const std::string& so_path, const std::string& func_name, bool throw_if_not_found = true) { - void* so_handle; - ORT_ENFORCE(Env::Default().LoadDynamicLibrary(so_path, false, &so_handle).IsOK()); - void* func = nullptr; - Status s = Env::Default().GetSymbolFromLibrary(so_handle, func_name, &func); - if (throw_if_not_found && !s.IsOK()) - ORT_ENFORCE(false, "Cannot find ", func_name, " in ", so_path); - return func; -} - -static void ParseVersion(const char* version, int* major, int* minor, int* patch) { - std::stringstream ss(version); - std::string val; - - auto ver_num_fn = [](const std::string& val) { - ORT_ENFORCE(!val.empty(), "Empty version number"); - if (val.length() > 1 && val[0] == '0') { - ORT_THROW("Invalid version number: ", val); - } - ORT_ENFORCE(std::all_of(val.begin(), val.end(), [](char c) { return isdigit(c); }), - "Invalid version number: ", val); - return std::stoi(val); - }; - - std::getline(ss, val, '.'); - ORT_ENFORCE(ss.good(), "Invalid version format: ", version); - *major = ver_num_fn(val); - - std::getline(ss, val, '.'); - *minor = ver_num_fn(val); - - std::getline(ss, val); - *patch = ver_num_fn(val); -} - -static bool VerifyCacheVersion(const std::string& so_path) { - // ensure we have _ORTInternal_GetCacheVersion_ function - void* f = GetFuncFromLibrary(so_path, "_ORTInternal_GetCacheVersion", /*throw_if_not_found*/ true); - if (f == nullptr) - return false; - - typedef const char* (*GetVersionFunc)(); - GetVersionFunc func = reinterpret_cast(f); - const char* cache_version = func(); - ORT_ENFORCE(cache_version, "Null cache version string!"); - int cur_major, cur_minor, cur_patch; - ParseVersion(__NUPHAR_CACHE_VERSION__, &cur_major, &cur_minor, &cur_patch); - int cache_major, cache_minor, cache_patch; - ParseVersion(cache_version, &cache_major, &cache_minor, &cache_patch); - - // make version check strict until we have thorough design for compatibility - bool version_match = (cur_major == cache_major) && (cur_minor == cache_minor); - if (!version_match) { - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Current nuphar runtime version (" << __NUPHAR_CACHE_VERSION__ << ") doesn't match cached dll version (" << cache_version << ")"; - } - return version_match; -} - -static bool VerifyTVMModuleChecksum(const std::string& so_path) { - void* f = GetFuncFromLibrary(so_path, "_ORTInternal_GetCheckSum", /*throw_if_not_found*/ false); - if (f) { - typedef void (*GetChecksumFunc)(const char*&, size_t&); - GetChecksumFunc func = reinterpret_cast(f); - const char* model_checksum; - size_t model_checksum_len; - func(model_checksum, - model_checksum_len); - - codegen::CodeGenSettings& setting = codegen::CodeGenSettings::Instance(); - // When checksum is expected by dll/so, user must set environment variable - // NUPHAR_CACHE_MODEL_CHECKSUM from md5 digest of running model. - // User may choose to run with base model or simplified mode and any match - // would be regarded as validated. - // Note that checksum validation here is not designed as a security measurement, - // so checksum compute is not done inside ORT. - if (setting.OptionMatches( - kNupharCacheModelChecksum, - std::string(model_checksum, model_checksum_len))) { - return true; - } else { - static std::mutex warn_mutex; - std::lock_guard warn_lock(warn_mutex); - static std::string last_warned_so_path; - if (last_warned_so_path != so_path) { - // warning only once for each so_path - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Cache checksum validation failed, using JIT..."; - } - return false; - } - } else { - // do not validate checksum if dll didn't require it (usually during debugging) - // TODO: force checksum validation in final release - return true; - } -} - -CacheStatus LoadTVMPackedFuncFromCache(const std::string& func_name, tvm::runtime::PackedFunc& func) { - std::string so_path; - if (!GetCacheSoFilePath(so_path)) { - if (codegen::CodeGenSettings::Instance().HasOption(kNupharCachePath)) { - return CacheStatus::Missing; - } else { - return CacheStatus::NotInUse; - } - } - - if (!VerifyCacheVersion(so_path)) { - return CacheStatus::Mismatch; - } - - if (!VerifyTVMModuleChecksum(so_path)) - return CacheStatus::Mismatch; - - tvm::runtime::Module module = tvm::runtime::Module::LoadFromFile(so_path); - func = module.GetFunction(func_name); - if (func == nullptr) { - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Cannot find " << func_name << " in cache, using JIT..."; - return CacheStatus::Missing; - } - return CacheStatus::Found; -} - -void SaveTVMModuleToCache(const std::string& filename, tvm::runtime::Module& module) { - fs::path path; - - static std::mutex save_cache_mutex; - std::lock_guard lock(save_cache_mutex); - if (GetOrCreateTVMModuleCacheDirectory(path, /*create*/ true)) { - path.append(filename + ".o"); - if (fs::exists(path)) { - //LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "Object file " << path << " already exists, skip saving..."; - return; - } - module->SaveToFile(path.string(), "o"); - } -} - -std::string GetPackedFuncName(const nuphar::NupharSubgraphUnit& subgraph, const CodeGenTarget& codegen_target, int64_t parallel_min_workloads) { - // in C, a function does not allow its name starting with a digit. - return NormalizeCppName("_" + subgraph.UniqueId() + "_" + codegen_target.GetTargetName() + "_p" + std::to_string(parallel_min_workloads)); -} - -bool TryCreateConstantScalar( - tvm::Expr& scalar, - const Tensor* tensor) { - if (!tensor) - return false; - - auto num_elements = tensor->Shape().Size(); - if (num_elements > 1) { - // for non-scalar, only fold to constant scalar when all values are identical - const auto& dtype = tensor->DataType(); - auto elem_size = dtype->Size(); - const void* data = tensor->DataRaw(); - -#define CHECK_ALL_TENSOR_SAME(T) \ - for (int64_t i = 1; i < num_elements; ++i) { \ - if (reinterpret_cast(data)[i] != reinterpret_cast(data)[0]) \ - return false; \ - } - - switch (elem_size) { - case 1: - CHECK_ALL_TENSOR_SAME(int8_t); - break; - case 2: - CHECK_ALL_TENSOR_SAME(int16_t); - break; - case 4: - CHECK_ALL_TENSOR_SAME(int32_t); - break; - case 8: - CHECK_ALL_TENSOR_SAME(int64_t); - break; - default: - return false; - } - -#undef CHECK_ALL_TENSOR_SAME - } - -#define ASSIGN_TVM_SCALAR(tvm_type, tensor_type) \ - if (tensor->IsDataType()) { \ - scalar = tvm::make_const(tvm_type, *tensor->Data()); \ - } - -#define ASSIGN_TVM_SCALAR_ELSE(tvm_type, tensor_type) \ - else ASSIGN_TVM_SCALAR(tvm_type, tensor_type) - - ASSIGN_TVM_SCALAR(HalideIR::Float(32), float) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::Float(64), double) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::Int(64), int64_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::Int(32), int32_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::Int(16), int16_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::Int(8), int8_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::UInt(64), uint64_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::UInt(32), uint32_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::UInt(16), uint16_t) - ASSIGN_TVM_SCALAR_ELSE(HalideIR::UInt(8), uint8_t) - else { - return false; - } - -#undef ASSIGN_TVM_SCALAR - - return true; -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h b/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h deleted file mode 100644 index a9f75bdf17..0000000000 --- a/onnxruntime/core/providers/nuphar/common/nuphar_tvm_utils.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include -#include - -#include "core/graph/graph.h" - -namespace onnxruntime { - -//forward -class CodeGenTarget; -class Tensor; - -namespace nuphar { - -struct NupharSubgraphUnit; //forward -// Helper functions to create or load from offline cached dll -// note after saving to obj file, we need to use tvm Python to create dll -// using script at onnxruntime/core/codegen/mti/scripts/create_shared.py - -enum class CacheStatus { - NotInUse, - Mismatch, - Missing, - Found, -}; - -CacheStatus LoadTVMPackedFuncFromCache(const std::string& func_name, tvm::runtime::PackedFunc& func); -void SaveTVMModuleToCache(const std::string& filename, tvm::runtime::Module& module); - -std::string GetPackedFuncName(const nuphar::NupharSubgraphUnit& subgraph, const CodeGenTarget& codegen_target, int64_t parallel_min_workloads); - -bool TryCreateConstantScalar(tvm::Expr& scalar, const Tensor* tensor); -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/utils.cc b/onnxruntime/core/providers/nuphar/common/utils.cc deleted file mode 100644 index a69a7547ee..0000000000 --- a/onnxruntime/core/providers/nuphar/common/utils.cc +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/common/utils.h" - -#include "core/framework/tensorprotoutils.h" -#include "core/providers/common.h" - -namespace onnxruntime { -namespace nuphar { - -bool NodeArgShapeUnknownOnAxis(const NodeArg* def, int64_t axis) { - auto shape = def->Shape(); - axis = HandleNegativeAxis(axis, shape->dim_size()); - ORT_ENFORCE(axis < shape->dim_size()); - auto dim = shape->dim(axis); - return utils::HasDimParam(dim) || (!utils::HasDimParam(dim) && !utils::HasDimValue(dim)); -} - -bool HasUnknownShapeOnAxis(const ConstPointerContainer>& defs, int64_t axis) { - for (const NodeArg* def : defs) { - if (NodeArgShapeUnknownOnAxis(def, axis)) { - return true; - } - } - return false; -} - -bool HasUnknownShapeOnAxes(const NodeArg* def, std::vector& axes) { - for (auto axis : axes) { - if (NodeArgShapeUnknownOnAxis(def, axis)) { - return true; - } - } - return false; -} - -Status GetVectorInt64FromTensorProto(std::vector& v, - const ONNX_NAMESPACE::TensorProto& tp) { - size_t tp_sz_in_bytes; - ORT_RETURN_IF_ERROR(utils::GetSizeInBytesFromTensorProto<0>(tp, &tp_sz_in_bytes)); - OrtValue ort_value; - std::unique_ptr data(new char[tp_sz_in_bytes]); - -#define UNPACK_TENSOR(T) \ - T* p = reinterpret_cast(data.get()); \ - ORT_RETURN_IF_ERROR(utils::UnpackTensor( \ - tp, \ - tp.raw_data().size() ? tp.raw_data().data() : nullptr, \ - tp.raw_data().size(), \ - p, \ - tp_sz_in_bytes / sizeof(T))); \ - std::vector tmp_v(p, p + tp_sz_in_bytes / sizeof(T)); - v.clear(); - - switch (tp.data_type()) { - case ONNX_NAMESPACE::TensorProto_DataType_INT32: { - UNPACK_TENSOR(int32_t); - for (auto axis : tmp_v) { - v.push_back(static_cast(axis)); - } - break; - } - case ONNX_NAMESPACE::TensorProto_DataType_INT64: { - UNPACK_TENSOR(int64_t); - v.insert(v.end(), tmp_v.begin(), tmp_v.end()); - break; - } - default: - ORT_NOT_IMPLEMENTED("Unimplemented type: ", tp.data_type()); - } - - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/common/utils.h b/onnxruntime/core/providers/nuphar/common/utils.h deleted file mode 100644 index 1b7939b4b0..0000000000 --- a/onnxruntime/core/providers/nuphar/common/utils.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/graph/graph.h" - -// forward declaration -struct OrtMemoryInfo; - -namespace onnxruntime { -namespace nuphar { - -bool NodeArgShapeUnknownOnAxis(const NodeArg* def, int64_t axis); - -bool HasUnknownShapeOnAxis(const ConstPointerContainer>& defs, int64_t axis); - -bool HasUnknownShapeOnAxes(const NodeArg* def, std::vector& axes); - -Status GetVectorInt64FromTensorProto(std::vector& v, - const ONNX_NAMESPACE::TensorProto& tp); - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/codegen_manager.cc b/onnxruntime/core/providers/nuphar/compiler/codegen_manager.cc deleted file mode 100644 index 9d15a12543..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/codegen_manager.cc +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/codegen_manager.h" - -#include "core/codegen/passes/op_ir_creator/all_ops.h" -#include "core/codegen/passes/scheduler/all_schedules.h" -#include "core/codegen/passes/weight_layout/transpose_2d.h" -#include "core/codegen/passes/weight_layout/tiling_2d.h" -#include "core/codegen/passes/weight_layout/vertical_stripes_2d.h" -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" -#include "core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h" - -namespace onnxruntime { -namespace codegen { -// explicit instantiation -template class RegistryBase; -} // namespace codegen - -namespace nuphar { - -//// All Creator instance registration -// 1. Create Customized Op IR creator instances - -// BEGIN: NupharTVM X86 IR creator classes - -#define ADD_OP_ITEM(name) \ - op_ir_registry->Register(std::move(std::make_unique())); - -#define POOL_OP(OP) ADD_OP_ITEM(OP) -#define REDUCE_V_OP(name) ADD_OP_ITEM(name) -#define UNARY_OP(name) ADD_OP_ITEM(name) - -static void RegisterAllNupharX86OpIRCreators(tvm_codegen::OpIRRegistry* op_ir_registry) { - LIST_ALL_X86_OPS() -} - -#undef ADD_OP_ITEM -#undef POOL_OP -#undef REDUCE_V_OP -#undef UNARY_OP - -// END: NupharTVM X86 IR creator classes - -// 2. Create Scheduler instances -// BEGIN: Nuphar Scheduler classes - -static void RegisterAllNupharSchedulers(tvm_codegen::TVMScheduleRegistry* sched_registry) { - // Add Generic TVM Rule schedules - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - - // Add Generic OpType schedules - sched_registry->Register( - std::move(std::make_unique())); - - // Add NupharX86 TVM Rule schedules - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - - // Add NupharX86 Tensorization schedules - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - - // Add NupharX86 OpType schedules - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - - // Add NupharX86 use count schedules - sched_registry->Register( - std::move(std::make_unique())); - sched_registry->Register( - std::move(std::make_unique())); - - // Add NupharX86 partial result schedules - sched_registry->Register( - std::move(std::make_unique())); -} - -// END: Nuphar Scheduler classes - -// 3. Create Weight layout instances -// BEGIN: Nuphar Weight Layouts classes -static void RegisterAllNupharWeightLayouts(tvm_codegen::WeightLayoutRegistry* layout_registry) { - // AVX512 - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8, 64))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16, 64))); - // AVX2 - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8, 32))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16, 32))); - // AVX - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8, 16))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16, 16))); - - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT, 8))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_FLOAT))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8))); - layout_registry->Register( - std::move(std::make_unique(ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT16))); -} - -// END: Nuphar Weight Layouts classes - -//// All Plugins for Nuphar provider -// 1. Plugin IR creator classes - -// BEGIN: Nuphar TVM X86 IR creator classes -#define ADD_OP_ITEM(name) \ - dispatcher->Register(#name, registry->Get(NUPHAR_TVM_X86_OP_IR_CREATOR_STRING(name))); - -#define POOL_OP(OP) ADD_OP_ITEM(OP) -#define REDUCE_V_OP(name) ADD_OP_ITEM(name) -#define UNARY_OP(name) ADD_OP_ITEM(name) - -static void RegisterNupharX86Dispatcher(const std::shared_ptr& builder, - const tvm_codegen::OpIRRegistry* registry) { - auto dispatcher = std::make_unique("OptypeNupharTVMX86Creators"); - LIST_ALL_X86_OPS() - builder->InsertDispatcher(std::move(dispatcher)); -} - -#undef ADD_OP_ITEM -#undef POOL_OP -#undef REDUCE_V_OP -#undef UNARY_OP -// END: Nuphar TVM X86 IR creator classes - -// 2 Plugin Scheduler classes - -// BEGIN: TVM rule Scheduler -static void RegisterNupharX86TVMRuleSchedulers(const std::shared_ptr& builder, - const tvm_codegen::TVMScheduleRegistry* registry) { - auto dispatcher = std::make_unique("NupharX86TVMRuleSchedulers"); - - // Register a scheduler for TVM External Tensor - dispatcher->Register(tvm_codegen::GetTVMOpRule(tvm_codegen::TVMOpRuleType::Extern), - registry->Get(TVM_SCHEDULER_STRING(Extern, NupharX86TVMRule))); - // Register a scheduler for TVM Reduce Tensor - dispatcher->Register(tvm_codegen::GetTVMOpRule(tvm_codegen::TVMOpRuleType::ComputeReduce), - registry->Get(TVM_SCHEDULER_STRING(Reduce, NupharX86TVMRule))); - - builder->InsertDispatcher(std::move(dispatcher)); -} -// END: TVM rule Scheduler - -// BEGIN: Tensorization Scheduler -static void RegisterNupharX86TensorizeScheduler(const std::shared_ptr& builder, - const tvm_codegen::TVMScheduleRegistry* registry) { - auto dispatcher = std::make_unique("NupharOrtTensorizeScheduler"); - - // Register a scheduler for Ort MatMulInteger - dispatcher->Register("MatMulInteger", - registry->Get(TVM_SCHEDULER_STRING(MatMulInteger, NupharX86Tensorize))); - builder->InsertDispatcher(std::move(dispatcher)); -} -// END: Tensorization Scheduler - -// BEGIN: ORT OpType Scheduler -static void RegisterNupharX86OrtOpTypeSchedulers(const std::shared_ptr& builder, - const tvm_codegen::TVMScheduleRegistry* registry) { - auto dispatcher = std::make_unique("NupharX86OrtOpTypeSchedulers"); - - // Register a scheduler for Ort Softmax OpType - dispatcher->Register("Softmax", - registry->Get(TVM_SCHEDULER_STRING(Softmax, NupharX86OrtOpType))); - - dispatcher->Register("Split", - registry->Get(TVM_SCHEDULER_STRING(Split, NupharX86OrtOpType))); - - builder->InsertDispatcher(std::move(dispatcher)); -} -// END: ORT OpType Scheduler - -// BEGIN: Reuse Count Analysis Scheduler -static void RegisterNupharX86UseCountSchedulers(const std::shared_ptr& builder, - const tvm_codegen::TVMScheduleRegistry* registry) { - auto dispatcher = std::make_unique("NupharX86UseCountSchedulers"); - - // Register a scheduler for Reuse count > 1 - dispatcher->Register("True", - registry->Get(TVM_SCHEDULER_STRING(True, NupharX86UseCount))); - - // Register a scheduler for Reuse count <= 1 - dispatcher->Register("False", - registry->Get(TVM_SCHEDULER_STRING(False, NupharX86UseCount))); - - builder->InsertDispatcher(std::move(dispatcher)); -} -// END: Reuse Count Analysis Scheduler - -// BEGIN: Partial Result Scheduler -static void RegisterNupharX86PartialResultSchedulers(const std::shared_ptr& builder, - const tvm_codegen::TVMScheduleRegistry* registry) { - auto dispatcher = std::make_unique("NupharX86PartialResultSchedulers"); - dispatcher->Register("True", - registry->Get(TVM_SCHEDULER_STRING(True, NupharX86PartialResult))); - - builder->InsertDispatcher(std::move(dispatcher)); -} -// END: Partial Result Scheduler - -TVMCodeGenManager::TVMCodeGenManager() { - op_ir_registry_ = std::make_unique(); - layout_registry_ = std::make_unique(); - schedule_registry_ = std::make_unique(); -} - -void TVMCodeGenManager::Initialization() { - RegisterAllNupharX86OpIRCreators(op_ir_registry_.get()); - RegisterAllGenericOpIRCreators(op_ir_registry_.get()); - - RegisterAllNupharWeightLayouts(layout_registry_.get()); - RegisterAllNupharSchedulers(schedule_registry_.get()); -} - -// TODO Add isa support -void TVMCodeGenManager::SetCodeGenHandle(NupharCodeGenHandle* handle) { - // layout registry - handle->layout_registry = layout_registry_.get(); - - // Op IR creators - handle->op_ir_builder = - std::make_shared("Nuphar_Op_IR_Builder"); - RegisterNupharX86Dispatcher(handle->op_ir_builder, op_ir_registry_.get()); - RegisterGenericOrtOpTypeDispatcher(handle->op_ir_builder, op_ir_registry_.get()); - - // Schedulers - handle->schedule_builder = - std::make_shared("Nuphar_Schedule_Builder"); - - RegisterNupharX86TVMRuleSchedulers(handle->schedule_builder, schedule_registry_.get()); - RegisterNupharX86TensorizeScheduler(handle->schedule_builder, schedule_registry_.get()); - RegisterNupharX86OrtOpTypeSchedulers(handle->schedule_builder, schedule_registry_.get()); - RegisterNupharX86UseCountSchedulers(handle->schedule_builder, schedule_registry_.get()); - RegisterNupharX86PartialResultSchedulers(handle->schedule_builder, schedule_registry_.get()); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/codegen_manager.h b/onnxruntime/core/providers/nuphar/compiler/codegen_manager.h deleted file mode 100644 index 75a88002fe..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/codegen_manager.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/passes/op_ir_creator/tvm_op_creator.h" -#include "core/codegen/passes/op_ir_creator/tvm_ir_builder.h" -#include "core/codegen/passes/scheduler/tvm_schedule_builder.h" -#include "core/codegen/passes/weight_layout/weight_layout.h" -#include "core/providers/nuphar/compiler/nuphar_handle.h" - -namespace onnxruntime { -namespace nuphar { - -// TVMCodeGenManager contains all registries -// including 1) TVM IR builder registry -// 2) Weight layout transformer registry -// 3) TVM scheduler registry, etc. -// These registries include all applicable passes for specific arch -// AND might also include non-applicable passes, like passes for another arch. - -// TVMCodeGenManager keeps the ownerships of all registries, passes, -// and planners. - -// TVMCodeGenManager also sets NupharCodeGenHandle for a specific arch. - -class TVMCodeGenManager { - public: - TVMCodeGenManager(); - - // TODO add a list of condition to handle dynamic registration - void Initialization(); - - // TODO: add target as an input - void SetCodeGenHandle(NupharCodeGenHandle* handle); - - private: - std::unique_ptr op_ir_registry_; - std::unique_ptr layout_registry_; - std::unique_ptr schedule_registry_; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/func_info.cc b/onnxruntime/core/providers/nuphar/compiler/func_info.cc deleted file mode 100644 index 776686eec2..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/func_info.cc +++ /dev/null @@ -1,570 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/func_info.h" - -#include "core/providers/nuphar/runtime/control_flow/scan_exec_ctx.h" -#include "core/framework/op_kernel.h" -#include "core/framework/tensorprotoutils.h" -#include "core/framework/onnxruntime_typeinfo.h" -#include "core/codegen/common/common.h" -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" -#include - -namespace onnxruntime { -namespace nuphar { - -static void FillBasicFuncInfo(NupharFuncInfo* func_info, - nuphar::OrtSubgraphAllocationInfo* partition_info, - const nuphar::NupharSubgraphUnit& subgraph, - const NupharCodeGenCtx& codegen_ctx, - tvm::Target tvm_target, - tvm::runtime::PackedFunc packed_func, - const std::string& name) { - ORT_ENFORCE(nullptr != func_info); - ORT_ENFORCE(nullptr != partition_info); - - func_info->name = name; - func_info->packed_func = packed_func; - func_info->device_type = static_cast(tvm_target->device_type); - - int tvm_input_idx = 0; - int def_index = 0; - // Handle inputs - func_info->ort_input_count = subgraph.inputs.size(); - // Assign Input meta - for (auto& def : subgraph.inputs) { - // fill in allocator info - NupharFuncInfo::AllocatorMeta input_allocator; - if (partition_info->inputs.count(def->Name()) > 0) { - // if an input is from external - input_allocator.index = partition_info->inputs.at(def->Name()); - input_allocator.is_external = true; - func_info->ort_input_allocator_is_collided_output.push_back(false); - } else if (partition_info->outputs.count(def->Name()) > 0) { - // if an input is from a previous real output - input_allocator.index = partition_info->outputs.at(def->Name()); - input_allocator.is_external = true; // a real output is always from external - func_info->ort_input_allocator_is_collided_output.push_back(true); - } else { - // else, an input is from an internal allocator - input_allocator.index = partition_info->CreateOrGetInternalAllocatorOffset(def->Name()); - input_allocator.is_external = false; - func_info->ort_input_allocator_is_collided_output.push_back(false); - } - - func_info->ort_input_allocators.push_back(input_allocator); - - if (codegen_ctx.IsInitializer(def->Name())) { - ++def_index; - continue; // skip initializers - } - - // fill in func args - NupharFuncInfo::FuncArgMeta input_meta; - input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); - input_meta.ort_arg_index = def_index; - - // fill in shape info and symobolic info - for (int dim = 0; dim < gsl::narrow(ShapeRank(def)); ++dim) { - if (ShapeHasSymbol(def, dim)) { - input_meta.inferred_shape.push_back(Dimension_Unknown); - input_meta.dim_symbols.push_back(std::make_pair(gsl::narrow(dim), ShapeSymbol(def, dim))); - } else if (ShapeHasValue(def, dim)) { - input_meta.inferred_shape.push_back(ShapeValue(def, dim)); - } else { - input_meta.inferred_shape.push_back(Dimension_Unknown); - } - } - - func_info->input_metas.push_back(input_meta); - - ++tvm_input_idx; - ++def_index; - } - - // Handle initializers - // Initializer meta - std::vector& intializers = func_info->intializers; - // Assign Initializer meta - for (const auto& item : codegen_ctx.GetWeightLayoutMap()) { - const WeightLayoutCodegenInfo* layout_info = item.second.get(); - bool is_marshalled = layout_info->is_marshalled; - const Tensor* t = - is_marshalled ? layout_info->marshalled_initializer - : codegen_ctx.GetOrtInitializerTensor(item.first); - - intializers.push_back(t); - ++tvm_input_idx; - } - - // set input_count = the number of inputs + the number of initializers - func_info->func_input_count = gsl::narrow(tvm_input_idx); - - // Handle Outputs - - func_info->ort_output_count = subgraph.outputs.size(); - // Assign Output meta - int tvm_output_idx = 0; - std::unordered_map visited_output_def_indices; - def_index = 0; - for (auto& def : subgraph.outputs) { - // fill in allocator info - NupharFuncInfo::AllocatorMeta output_allocator; - if (partition_info->outputs.count(def->Name()) > 0) { - // if an output is from external - output_allocator.index = partition_info->outputs.at(def->Name()); - output_allocator.is_external = true; - } else { - // else, an output is from an internal allocator - output_allocator.index = partition_info->CreateOrGetInternalAllocatorOffset(def->Name()); - output_allocator.is_external = false; - } - - func_info->ort_output_allocators.push_back(output_allocator); - - // check output alias - const NodeArg* source_def = Promote(codegen_ctx.GetGraphStats()) - ->SourceDefOfOutputAlias(def); - - if (nullptr != source_def) { - // if def is an alias - auto key = GetKey(source_def); - if (visited_output_def_indices.count(key) != 0) { - // source_def has visisted ==> def is a duplicated output - // record the pair (dst of ort arg index, src of tvm func index) - func_info->ort_aliased_output_to_func_indices.emplace_back(def_index, - func_info->func_input_count + - visited_output_def_indices[key]); - - ++def_index; - continue; - } - // update visited_output_def_indices - visited_output_def_indices.insert(std::make_pair(key, gsl::narrow_cast(tvm_output_idx))); - } else { - auto key = GetKey(def); - if (visited_output_def_indices.count(key) != 0) { - // def has visisted ==> def is a duplicated output - // record the pair (dst of ort arg index, src of tvm func index) - func_info->ort_aliased_output_to_func_indices.emplace_back(def_index, - func_info->func_input_count + - visited_output_def_indices[key]); - - ++def_index; - continue; - } - visited_output_def_indices.insert(std::make_pair(key, gsl::narrow_cast(tvm_output_idx))); - } - - NupharFuncInfo::FuncArgMeta output_meta; - output_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); - output_meta.ort_arg_index = def_index; - - // fill in shape info and symobolic info - for (int dim = 0; dim < gsl::narrow(ShapeRank(def)); ++dim) { - if (ShapeHasSymbol(def, dim)) { - auto p = std::make_pair(gsl::narrow(dim), ShapeSymbol(def, dim)); - output_meta.dim_symbols.push_back(p); - output_meta.inferred_shape.push_back(Dimension_Unknown); - } else if (ShapeHasValue(def, dim)) { - output_meta.inferred_shape.push_back(ShapeValue(def, dim)); - } else { - output_meta.inferred_shape.push_back(Dimension_Unknown); - } - } - - func_info->output_metas.push_back(output_meta); - ++def_index; - ++tvm_output_idx; - } - - // set output_count as the real output count - func_info->func_output_count = gsl::narrow_cast(tvm_output_idx); - - // set tvm type_codes - func_info->type_codes.resize(func_info->func_input_count + func_info->func_output_count, TVMTypeCode::kNDArrayContainer); -} - -static void FillScanExecInfo(NupharFuncInfo* func_info, - nuphar::OrtSubgraphAllocationInfo* partition_info, - const Node& node, - const NupharCodeGenCtx& codegen_ctx, - tvm::Target tvm_target, - tvm::runtime::PackedFunc packed_func, - const std::string& name) { - ORT_ENFORCE(nullptr != func_info); - ORT_ENFORCE(nullptr != partition_info); - - // create Scan control-flow info - auto scan_info = std::make_unique(); - - int64_t num_state_variables; - int64_t num_scan_inputs; - int64_t num_scan_outputs; - - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper attrs(&ctx); - - // extract num_scan_inputs - bool attr_is_ok = attrs.GetAttr("num_scan_inputs", &num_scan_inputs).IsOK(); - ORT_UNUSED_PARAMETER(attr_is_ok); - ORT_ENFORCE_DEBUG(attr_is_ok); - - auto subgraph = GetSubgraph(node); - ORT_ENFORCE(subgraph != nullptr); - size_t num_variadic_inputs = subgraph->GetInputs().size(); - size_t num_variadic_outputs = subgraph->GetOutputs().size(); - - num_state_variables = gsl::narrow(num_variadic_inputs) - num_scan_inputs; - num_scan_outputs = gsl::narrow(num_variadic_outputs) - num_state_variables; - - // Set ScanExecInfo's parameter count meta - scan_info->num_state_variables = num_state_variables; - scan_info->num_scan_inputs = num_scan_inputs; - scan_info->num_scan_outputs = num_scan_outputs; - scan_info->num_scan_implicit_inputs = gsl::narrow_cast(node.ImplicitInputDefs().size()); - - // ScanExecInfo's control flow Meta - std::vector& scan_input_forwards = scan_info->scan_input_forwards; - std::vector& scan_output_forwards = scan_info->scan_output_forwards; - std::vector& scan_input_axes = scan_info->scan_input_axes; - std::vector& scan_output_axes = scan_info->scan_output_axes; - - scan_input_forwards.resize(num_scan_inputs); - scan_output_forwards.resize(num_scan_outputs); - - // extract directions and axes - std::vector scan_input_directions; - std::vector scan_output_directions; - - // scan_input_directions - if (attrs.GetAttrs("scan_input_directions", scan_input_directions).IsOK()) { - ORT_ENFORCE(gsl::narrow_cast(scan_input_directions.size()) == num_scan_inputs, - "Number of entries in 'scan_input_directions ' was ", scan_input_directions.size(), - ". Must match 'num_scan_inputs' of ", num_scan_inputs); - ORT_ENFORCE(std::all_of(scan_input_directions.cbegin(), scan_input_directions.cend(), - [](int64_t i) { return i == 0 || - i == 1; }), - "Invalid values in 'scan_input_directions'. 0 == forward. 1 == reverse."); - } else { - // default to forward - scan_input_directions = std::vector(num_scan_inputs, 0); - } - - // scan_input_forwards - for (size_t i = 0; i < gsl::narrow(num_scan_inputs); ++i) { - scan_input_forwards[i] = scan_input_directions[i] == 0; - } - - // scan_output_directions - if (attrs.GetAttrs("scan_output_directions", scan_output_directions).IsOK()) { - ORT_ENFORCE(gsl::narrow_cast(scan_output_directions.size()) == num_scan_outputs, - "Number of entries in 'scan_output_directions ' was ", scan_output_directions.size(), - ". Must match 'num_scan_outputs' of ", num_scan_outputs); - ORT_ENFORCE(std::all_of(scan_output_directions.cbegin(), scan_output_directions.cend(), - [](int64_t i) { return i == 0 || - i == 1; }), - "Invalid values in 'scan_output_directions'. 0 == forward. 1 == reverse."); - } else { - // default to forward - scan_output_directions = std::vector(num_scan_outputs, 0); - } - - // scan_output_forwards - for (size_t i = 0; i < gsl::narrow(num_scan_outputs); ++i) { - scan_output_forwards[i] = scan_output_directions[i] == 0; - } - - // scan_input_axes - if (attrs.GetAttrs("scan_input_axes", scan_input_axes).IsOK()) { - ORT_ENFORCE(gsl::narrow_cast(scan_input_axes.size()) == num_scan_inputs, - "Number of entries in 'scan_input_axes ' was ", scan_input_axes.size(), - ". Must match 'num_scan_inputs' of ", num_scan_inputs); - - } else { - // default to axis 0 - scan_input_axes = std::vector(num_scan_inputs, 0); - } - - // scan_output_axes - if (attrs.GetAttrs("scan_output_axes", scan_output_axes).IsOK()) { - ORT_ENFORCE(gsl::narrow_cast(scan_output_axes.size()) == num_scan_outputs, - "Number of entries in 'scan_output_axes ' was ", scan_output_axes.size(), - ". Must match 'num_scan_outputs' of ", num_scan_outputs); - - } else { - // default to axis 0 - scan_output_axes = std::vector(num_scan_outputs, 0); - } - - // handle NupharFuncInfo - func_info->name = name; - func_info->packed_func = packed_func; - func_info->device_type = static_cast(tvm_target->device_type); - - int tvm_input_idx = 0; - // Handle state inputs & inputs - func_info->ort_input_count = num_variadic_inputs; - - // assign state inputs & inputs - for (size_t ort_input_idx = 0; ort_input_idx < num_variadic_inputs; ++ort_input_idx) { - // fill in allocator info - NupharFuncInfo::AllocatorMeta input_allocator; - const NodeArg* main_graph_def = node.InputDefs()[ort_input_idx]; - ORT_ENFORCE(nullptr != main_graph_def); - if (partition_info->inputs.count(main_graph_def->Name()) > 0) { - // if an input is from external - input_allocator.index = partition_info->inputs.at(main_graph_def->Name()); - input_allocator.is_external = true; - func_info->ort_input_allocator_is_collided_output.push_back(false); - } else if (partition_info->outputs.count(main_graph_def->Name()) > 0) { - // if an input is from a previous real output - input_allocator.index = partition_info->outputs.at(main_graph_def->Name()); - input_allocator.is_external = true; // a real output is always from external - func_info->ort_input_allocator_is_collided_output.push_back(true); - } else { - // else, an input is from an internal allocator - input_allocator.index = partition_info->CreateOrGetInternalAllocatorOffset(main_graph_def->Name()); - input_allocator.is_external = false; - func_info->ort_input_allocator_is_collided_output.push_back(false); - } - - func_info->ort_input_allocators.push_back(input_allocator); - - const NodeArg* def = subgraph->GetInputs()[ort_input_idx]; - ORT_ENFORCE(nullptr != def); - - if (ort_input_idx >= gsl::narrow(num_state_variables)) { - // initializer should only happen in real inputs, not in state inputs - if (codegen_ctx.IsInitializer(def->Name())) { - continue; // skip initializers - } - } - - NupharFuncInfo::FuncArgMeta input_meta; - input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); - input_meta.ort_arg_index = gsl::narrow_cast(ort_input_idx); - - // fill in shape info and symobolic info - for (int dim = 0; dim < gsl::narrow(ShapeRank(def)); ++dim) { - if (ShapeHasSymbol(def, dim)) { - auto p = std::make_pair(gsl::narrow(dim), ShapeSymbol(def, dim)); - input_meta.dim_symbols.push_back(p); - input_meta.inferred_shape.push_back(Dimension_Unknown); - } else if (ShapeHasValue(def, dim)) { - input_meta.inferred_shape.push_back(ShapeValue(def, dim)); - } else { - input_meta.inferred_shape.push_back(Dimension_Unknown); - } - } - - func_info->input_metas.push_back(input_meta); - ++tvm_input_idx; - } - - size_t ort_input_idx = num_variadic_inputs; - // Handle implicit inputs - for (const NodeArg* def : node.ImplicitInputDefs()) { - NupharFuncInfo::AllocatorMeta input_allocator; - if (partition_info->inputs.count(def->Name()) > 0) { - // if an input is from external - input_allocator.index = partition_info->inputs.at(def->Name()); - input_allocator.is_external = true; - func_info->ort_input_allocator_is_collided_output.push_back(false); - } else if (partition_info->outputs.count(def->Name()) > 0) { - // if an input is from a previous real output - input_allocator.index = partition_info->outputs.at(def->Name()); - input_allocator.is_external = true; - func_info->ort_input_allocator_is_collided_output.push_back(true); - } else { - // else, an input is from an internal allocator - input_allocator.index = partition_info->CreateOrGetInternalAllocatorOffset(def->Name()); - input_allocator.is_external = false; - func_info->ort_input_allocator_is_collided_output.push_back(false); - } - - func_info->ort_input_allocators.push_back(input_allocator); - - // skip initializers - if (codegen_ctx.IsInitializer(def->Name())) { - ++ort_input_idx; - continue; // skip initializers - } - - NupharFuncInfo::FuncArgMeta input_meta; - input_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); - input_meta.ort_arg_index = gsl::narrow_cast(ort_input_idx); - - std::vector> symbols; - for (int dim = 0; dim < gsl::narrow(ShapeRank(def)); ++dim) { - if (ShapeHasSymbol(def, dim)) { - auto p = std::make_pair(gsl::narrow(dim), ShapeSymbol(def, dim)); - input_meta.dim_symbols.push_back(p); - input_meta.inferred_shape.push_back(Dimension_Unknown); - } else if (ShapeHasValue(def, dim)) { - input_meta.inferred_shape.push_back(ShapeValue(def, dim)); - } else { - input_meta.inferred_shape.push_back(Dimension_Unknown); - } - } - func_info->input_metas.push_back(input_meta); - ++tvm_input_idx; - ++ort_input_idx; - } - - // Handle initializers - // Initializer meta - std::vector& intializers = func_info->intializers; - - // Assign Initializer meta - for (const auto& item : codegen_ctx.GetWeightLayoutMap()) { - const WeightLayoutCodegenInfo* layout_info = item.second.get(); - - bool is_marshalled = layout_info->is_marshalled; - const Tensor* t = - is_marshalled ? layout_info->marshalled_initializer - : codegen_ctx.GetOrtInitializerTensor(item.first); - - intializers.push_back(t); - ++tvm_input_idx; - } - - // set input_count = the number of inputs (real inputs + state inputs) + the number of initializers - func_info->func_input_count = gsl::narrow(tvm_input_idx); - - // Handle State Outputs and Outputs - func_info->ort_output_count = num_variadic_outputs; - - // Since in Scan, we only allow state using output's memory during Execution, not the other around. - // When one input and one state are aliased, the kept one can only be the input. - // Therefore, we do alias detection starting from inputs first. - std::unordered_map visited_output_def_indices; - for (size_t ort_output_idx = gsl::narrow(num_state_variables); ort_output_idx < num_variadic_outputs; ++ort_output_idx) { - const NodeArg* def = subgraph->GetOutputs()[ort_output_idx]; - ORT_ENFORCE(nullptr != def); - const NodeArg* source_def = Promote(codegen_ctx.GetGraphStats()) - ->SourceDefOfOutputAlias(def); - if (nullptr != source_def) { - auto key = GetKey(source_def); - ORT_ENFORCE(visited_output_def_indices.count(key) == 0, - "Scan has alias btw two inputs. Nuphar only support aliasing btw state and output in Scan"); - visited_output_def_indices.insert(std::make_pair(key, gsl::narrow(ort_output_idx))); - } else { - auto key = GetKey(def); - visited_output_def_indices.insert(std::make_pair(key, gsl::narrow(ort_output_idx))); - } - } - - // assign state outputs and outputs - size_t tvm_output_idx = 0; - std::unordered_map visited_output_state_func_indices; - for (size_t ort_output_idx = 0; ort_output_idx < num_variadic_outputs; ++ort_output_idx) { - // fill in allocator info - NupharFuncInfo::AllocatorMeta output_allocator; - const NodeArg* main_graph_def = node.OutputDefs()[ort_output_idx]; - ORT_ENFORCE(nullptr != main_graph_def); - if (partition_info->outputs.count(main_graph_def->Name()) > 0) { - output_allocator.index = partition_info->outputs.at(main_graph_def->Name()); - output_allocator.is_external = true; - } else { - output_allocator.index = partition_info->CreateOrGetInternalAllocatorOffset(main_graph_def->Name()); - output_allocator.is_external = false; - } - func_info->ort_output_allocators.push_back(output_allocator); - - // perform alias analysis - const NodeArg* def = subgraph->GetOutputs()[ort_output_idx]; - ORT_ENFORCE(nullptr != def); - const NodeArg* source_def = Promote(codegen_ctx.GetGraphStats()) - ->SourceDefOfOutputAlias(def); - - // Determine alias btw output and state output - auto key = source_def != nullptr ? GetKey(source_def) : GetKey(def); - - int ort_arg_index = gsl::narrow_cast(ort_output_idx); - if (ort_output_idx < gsl::narrow(num_state_variables)) { - auto key_iter = visited_output_def_indices.find(key); - // if ort_output_idx is a state output - if (key_iter != visited_output_def_indices.end()) { - // If state output is an alias - - auto output_tvm_idx = key_iter->second - gsl::narrow_cast(num_state_variables); - - // also record ort_aliased_output_to_func_indices - func_info->ort_aliased_output_to_func_indices.push_back( - std::make_pair(gsl::narrow(ort_output_idx), func_info->func_input_count + output_tvm_idx)); - - scan_info->state_to_output_indices.push_back(output_tvm_idx); - - if (visited_output_state_func_indices.count(key) != 0) { - // We could have multiple states that alias to the same output. - // We only record the first one and skip the rest. - continue; - } else { - // record i_output for the lookup of the aliased output later - visited_output_state_func_indices.insert( - std::make_pair(key, gsl::narrow(func_info->func_input_count + output_tvm_idx))); - - // override ort_arg_index using the output index - ort_arg_index = visited_output_def_indices[key]; - } - } else { - // the state output not aliased(no scan output shares with it) - scan_info->state_to_output_indices.push_back(NupharFuncInfo::Index_NonAliasedOutput); - } - } else { - // if ort_output_idx is an output - if (visited_output_state_func_indices.count(key) != 0) { - // skip a duplicated output, since it was counted in the duplicated state output previously - continue; - } - } - - NupharFuncInfo::FuncArgMeta output_meta; - output_meta.dtype = OrtTypeInfo::ElementTypeFromProto(def->TypeAsProto()->tensor_type().elem_type()); - output_meta.ort_arg_index = ort_arg_index; - - // shape and symbols - for (int dim = 0; dim < gsl::narrow(ShapeRank(def)); ++dim) { - if (ShapeHasSymbol(def, dim)) { - auto p = std::make_pair(gsl::narrow(dim), ShapeSymbol(def, dim)); - output_meta.dim_symbols.push_back(p); - output_meta.inferred_shape.push_back(Dimension_Unknown); - } else if (ShapeHasValue(def, dim)) { - output_meta.inferred_shape.push_back(ShapeValue(def, dim)); - } else { - output_meta.inferred_shape.push_back(Dimension_Unknown); - } - } - func_info->output_metas.push_back(output_meta); - ++tvm_output_idx; - } - - // set output_count as the real output count - func_info->func_output_count = tvm_output_idx; - - // set tvm type_codes - func_info->type_codes.resize(func_info->func_input_count + func_info->func_output_count, TVMTypeCode::kNDArrayContainer); - - // set control-flow info - func_info->cf_info = std::move(scan_info); -} - -void FillNupharFuncInfo(NupharFuncInfo* func_info, - nuphar::OrtSubgraphAllocationInfo* partition_info, - const nuphar::NupharSubgraphUnit& subgraph, - const NupharCodeGenCtx& codegen_ctx, - tvm::Target tvm_target, - tvm::runtime::PackedFunc packed_func, - const std::string& name) { - if (subgraph.nodes.front()->OpType() == "Scan") { - FillScanExecInfo(func_info, partition_info, *subgraph.nodes.front(), codegen_ctx, tvm_target, packed_func, name); - return; - } - - FillBasicFuncInfo(func_info, partition_info, subgraph, codegen_ctx, tvm_target, packed_func, name); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/func_info.h b/onnxruntime/core/providers/nuphar/compiler/func_info.h deleted file mode 100644 index 6b2780e665..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/func_info.h +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/common/common.h" -#include "core/common/common.h" -#include "core/framework/data_types.h" -#include "core/framework/tensor.h" -#include "core/graph/graph.h" -#include "core/providers/nuphar/common/nuphar_subgraph.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" - -#include -#include -#include - -namespace onnxruntime { -namespace nuphar { - -enum class ControlFlowInfoType : unsigned int { - Scan = 1, -}; - -// abstract class for control flow info -struct ControlFlowInfo { - private: - ControlFlowInfoType type; - - public: - ControlFlowInfo(ControlFlowInfoType _type) : type(_type) {} - - virtual ~ControlFlowInfo() = default; - - DYN_PROMOTE_BASE(ControlFlowInfo, ControlFlowInfoType, type) -}; - -// Add Promote support for ControlFlowInfo -// Note here we need to use DYN_PROMOTE instread of DYNAMIC_PROMOTE -// since ControlFlowInfo is a critical path -DYN_PROMOTE(ControlFlowInfo) - -// NupharFuncInfo holds tvm::runtime::PackedFunc (the generated function) -// And corresponding static meta information to call it, like number of argument and offset -// Note NupharFuncInfo includes ONLY parameters from codegen -// but DOES NOT include any runtime information. - -// The owner of NupharFuncInfo is currently NupharKernelState. -// NupharFuncInfo is created in NupharCompiler and is consumed by ExecBlock -// Note all of vectors use numbers of PackedFunc's parameters as vector bounds -// (meaning vector.size() == numbers of PackedFunc's parameters) -// except those denoted with ort, which use numbers of ort op's parameters as vector bounds. -// -1 might be inserted a bubble to keep positions and sizes for later lookup. -struct NupharFuncInfo { - // speicial value for *_func_indices - enum : int { - Index_NonAliasedOutput = -1, - }; - - // PackedFunc name - std::string name; - - // PackedFunc - tvm::runtime::PackedFunc packed_func; - - // TVM DLDevice - DLDeviceType device_type; - - struct FuncArgMeta { - MLDataType dtype; - // shapes with dimensions statically know or inferred at compile time - // symbolic dim would have Dimension_Unknown and will be patched at runtime - std::vector inferred_shape; - std::vector> dim_symbols; - int ort_arg_index; - }; - - std::vector input_metas; - std::vector output_metas; - std::vector> ort_aliased_output_to_func_indices; // A pair of (Ort dst index, TVM src index) - - struct AllocatorMeta { - int index; - bool is_external; - }; - - std::vector ort_input_allocators; - std::vector ort_output_allocators; - - // Note an input can be also an external output. - // It is due to NodeArg can be used by Nodes in - // and out of a subgraph at the same time. - // When it happens, we need to label it as a collided output, - // and record that external output allocator index. - std::vector ort_input_allocator_is_collided_output; - - // initializers meta - std::vector intializers; - - // Note the total arg number == input_count + output_count - size_t func_input_count; // input_count == real inputs + initializers - size_t func_output_count; // real outputs - - // tvm args (including input and outputs ) - std::vector type_codes; - - // control-flow info for the generated function - std::unique_ptr cf_info; - - size_t ort_input_count; - size_t ort_output_count; -}; - -void FillNupharFuncInfo(NupharFuncInfo* func_info, - nuphar::OrtSubgraphAllocationInfo* partition_info, - const nuphar::NupharSubgraphUnit& subgraph, - const NupharCodeGenCtx& codegen_ctx, - tvm::Target tvm_target, - tvm::runtime::PackedFunc packed_func, - const std::string& name); - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/initializer_info.h b/onnxruntime/core/providers/nuphar/compiler/initializer_info.h deleted file mode 100644 index b2fd478298..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/initializer_info.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/framework/tensor.h" -#include - -// TODO: move to nuphar -namespace onnxruntime { -namespace nuphar { - -// TODO: move it to weight layout place -struct WeightLayoutCodegenInfo { - const Tensor* marshalled_initializer = nullptr; // TODO: change it to unique_ptr - std::string layout = ""; // layout name - tvm::Tensor marshalled_tensor; - tvm::Tensor unmarshalled_tensor; - bool is_marshalled; - - WeightLayoutCodegenInfo(const tvm::Tensor& tvm_tensor) - : marshalled_tensor(tvm_tensor), unmarshalled_tensor(tvm_tensor), is_marshalled(false) {} -}; - -struct InitializerInfo { - const Tensor* original_initializer = nullptr; // original ort tensor - std::unique_ptr layout_info = nullptr; - - InitializerInfo(const Tensor* tensor) : original_initializer(tensor) {} -}; - -using InitializerMap = std::map; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc deleted file mode 100644 index 530b7602af..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.cc +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "nuphar_codegen_ctx.h" - -#include "core/codegen/common/common.h" -#include "core/codegen/common/utils.h" -#include "core/codegen/mti/mti_tvm_utils.h" // TODO: remove this after decoupling layout compile and run -#include "core/common/safeint.h" -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" -#include "core/codegen/passes/utils/ort_tvm_utils.h" // TODO: remove this after decoupling layout compile and run -#include // TODO: remove this after decoupling layout compile and run - -#include "core/providers/nuphar/common/nuphar_tvm_utils.h" - -namespace onnxruntime { -namespace nuphar { - -NupharCodeGenCtx::NupharCodeGenCtx( - const Node& node, - const std::map& initializers, - std::unordered_map>& global_generated_initializers, - const NupharCodeGenHandle* handle) - : CodeGenContext(handle), - nuphar_handle_(handle), - initializers_(initializers), - global_generated_initializers_(global_generated_initializers) { - // construct graph_stats - graph_stats_ = std::make_unique(nuphar_handle_->shape_inference); -} - -NupharCodeGenCtx::NupharCodeGenCtx( - const nuphar::NupharSubgraphUnit& subgraph, - std::unordered_map>& global_generated_initializers, - const NupharCodeGenHandle* handle) - : CodeGenContext(handle), - nuphar_handle_(handle), - initializers_(subgraph.initializers), - global_generated_initializers_(global_generated_initializers) { - graph_stats_ = std::make_unique(nuphar_handle_->shape_inference); - Promote(graph_stats_)->Evaluate(subgraph); -} - -// This is a temp function before we decouple weight layout compilation and run -// This will be moved. -// TODO: remove this. -static tvm::runtime::PackedFunc LowerLayoutFunc(const tvm_codegen::WeightLayout* layout) { - tvm::Array inputs; - tvm::Array outputs; - - layout->CreateLayoutMarshallingTVMOp(inputs, outputs); - - auto config = tvm::build_config(); - config->disable_select_rewriting = true; - auto S = tvm::create_schedule({outputs[0]->op}); - S[outputs[0]->op].compute_root(); - - std::string func_name = layout->Name() + "_marshall"; - - tvm::runtime::PackedFunc cached_func; - auto cache_status = nuphar::LoadTVMPackedFuncFromCache(func_name, cached_func); - if (cache_status != nuphar::CacheStatus::Found) { - ORT_ENFORCE(cached_func == nullptr); - auto lowered = tvm::lower(S, {inputs[0], outputs[0]}, func_name, {}, config); - auto module = tvm::build(lowered, tvm::target::llvm(), tvm::Target(), config); - tvm_codegen::DumpTVMModuleToFile(func_name, module); - if (cache_status == nuphar::CacheStatus::Missing) { - nuphar::SaveTVMModuleToCache(func_name, module); - } - cached_func = module.GetFunction(func_name); - } - return cached_func; -} - -// This is a temp function before we decouple weight layout compilation and run. -// This will be moved. -// TODO: remove this. -static const Tensor* Marshalling( - const std::string& initializer_name, - std::unordered_map>& global_generated_initializers, - const Tensor* original_initializer, - const tvm_codegen::WeightLayout* layout_ptr, - WeightLayoutCtx& ctx_layout, - AllocatorPtr allocator) { - tvm::runtime::PackedFunc packed_func; - - const std::string& layout_key = layout_ptr->Name(); - if (ctx_layout.weight_layout_to_packed_func.count(layout_key) == 0) { - packed_func = LowerLayoutFunc(layout_ptr); - ctx_layout.weight_layout_to_packed_func.insert(std::make_pair(layout_key, packed_func)); - } else { - packed_func = ctx_layout.weight_layout_to_packed_func[layout_key]; - } - - std::vector marshalled_shape = layout_ptr->ToActualShape(original_initializer); - auto marshalled_size = TotalSize(marshalled_shape); - auto byte_size = original_initializer->DataType()->Size(); - - std::unique_ptr out_ptr; - void* p_data = allocator->Alloc(SafeInt(marshalled_size) * byte_size); - out_ptr = std::make_unique( - original_initializer->DataType(), - TensorShape(marshalled_shape), - p_data, - allocator->Info()); - - global_generated_initializers.emplace(initializer_name, std::move(out_ptr)); - - int num_args = 2; - DLContext tvm_ctx{kDLCPU, 0}; - std::vector lvalues(num_args); - std::vector tvm_tensors(num_args); - - // input - const auto& tensor_shape = original_initializer->Shape(); - auto input_shape = tensor_shape.AsShapeVector(); - if (input_shape.empty()) - input_shape.push_back(1); - const void* input_data = original_initializer->DataRaw(); - DLDataType tvm_dtype = tvm_codegen::ToTvmDLDataType(original_initializer->DataType()); - - tvm_tensors[0] = {const_cast(input_data), tvm_ctx, - gsl::narrow_cast(input_shape.size()), tvm_dtype, - input_shape.data(), nullptr, 0}; - lvalues[0].v_handle = &(tvm_tensors[0]); - - // output - tvm_tensors[1] = {p_data, tvm_ctx, - gsl::narrow_cast(marshalled_shape.size()), tvm_dtype, - marshalled_shape.data(), nullptr, 0}; - lvalues[1].v_handle = &(tvm_tensors[1]); - - auto types_code = std::vector(num_args, kNDArrayContainer); - tvm::TVMArgs tvm_args(lvalues.data(), types_code.data(), num_args); - tvm::TVMRetValue rvalue; - packed_func.CallPacked(tvm_args, &rvalue); - return global_generated_initializers.at(initializer_name).get(); -} - -// on the fly WeightLayout transformer -tvm::Tensor NupharCodeGenCtx::ApplyWeightLayout( - const std::string& layout_key, - const std::string& initializer_name, - const tvm::Tensor& X, - bool returnMarshalled) { - tvm::Tensor marshalled; - ORT_ENFORCE(IsInitializer(initializer_name)); - auto layout_info = GetWeightLayoutInfo(initializer_name); - ORT_ENFORCE(nullptr != layout_info); - - const Tensor* original_initializer = GetOrtInitializerTensor(initializer_name); - - auto layout_ptr = nuphar_handle_->layout_registry->Get(layout_key); - ORT_ENFORCE(nullptr != layout_ptr); - - // check whether the weight is applied layout marshalling - if (nullptr == layout_info->marshalled_initializer) { - ORT_ENFORCE(!layout_info->is_marshalled); // initializer should not have been marshalled before - - // TODO: change to delayed call - layout_info->layout = layout_ptr->Name(); - - // TODO: change to delayed call - layout_info->marshalled_initializer = - Marshalling(initializer_name, - global_generated_initializers_, - original_initializer, - layout_ptr, - weight_layout_ctx_, - nuphar_handle_->allocator); - - layout_info->marshalled_tensor = tvm::placeholder(layout_ptr->ToActualShape(X), X->dtype, initializer_name + "_marshalled"); - layout_info->unmarshalled_tensor = tvm::compute( - X->shape, - [&](const tvm::Array& nominal_coord) { - tvm::Array cc; - for (auto v : nominal_coord) - cc.push_back(v); - - auto coord_trans_func = layout_ptr->ToActual(X); - return layout_info->marshalled_tensor(coord_trans_func(cc)); - }, - initializer_name + "_unmarshalled"); - - layout_info->is_marshalled = true; - - } else { - ORT_ENFORCE(layout_ptr->Name() == layout_info->layout); - } - - if (returnMarshalled) { - return layout_info->marshalled_tensor; - } - return layout_info->unmarshalled_tensor; -} - -const NupharSubgraphUnitStats* NupharCodeGenCtx::GetGraphStats() const { - return graph_stats_.get(); -} - -bool NupharCodeGenCtx::IsInitializer(const std::string& name) const { - return initializers_.count(name) > 0; -} - -const Tensor* NupharCodeGenCtx::GetOrtInitializerTensor(const std::string& name) const { - if (IsInitializer(name)) - return initializers_.at(name); - return nullptr; -} - -WeightLayoutCodegenInfo* NupharCodeGenCtx::GetWeightLayoutInfo(const std::string& name) { - if (initializer_layouts_.count(name) > 0) - return initializer_layouts_.at(name).get(); - return nullptr; -} - -const WeightLayoutCodegenInfo* NupharCodeGenCtx::GetWeightLayoutInfo(const std::string& name) const { - if (initializer_layouts_.count(name) > 0) - return initializer_layouts_.at(name).get(); - return nullptr; -} - -void NupharCodeGenCtx::CreateWeightLayoutInfo(const std::string& name, const tvm::Tensor& tensor) { - ORT_ENFORCE(initializer_layouts_.count(name) == 0); - initializer_layouts_.emplace(name, std::move(std::make_unique(tensor))); -} - -const std::map>& NupharCodeGenCtx::GetWeightLayoutMap() const { - return initializer_layouts_; -} - -void NupharCodeGenCtx::RecordTensorToNode(const tvm::Tensor& t, const Node* node) { - // Insert tvm::Tensor and Node to the lookup table - // But bypass it when node is a output alias - if (!Promote(graph_stats_)->IsOutputAlias(node)) - tvm_tensor_to_node_lookup_.insert(std::make_pair(t->op.get(), node)); -} - -const Node* NupharCodeGenCtx::FindNode(const tvm::Tensor& t) const { - auto p = tvm_tensor_to_node_lookup_.find(t->op.get()); - if (p != tvm_tensor_to_node_lookup_.end()) - return p->second; - return nullptr; -} - -const NupharCodeGenHandle* NupharCodeGenCtx::GetCodeGenHandle() const { - return nuphar_handle_; -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.h b/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.h deleted file mode 100644 index 354c7ca975..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_codegen_ctx.h +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/codegen/common/common.h" -#include "core/codegen/passes/utils/codegen_context.h" -#include "core/common/common.h" -#include "core/graph/graph.h" -#include "core/providers/nuphar/common/analysis/graph_stats.h" -#include "core/providers/nuphar/common/nuphar_subgraph.h" -#include "core/providers/nuphar/compiler/initializer_info.h" -#include "core/providers/nuphar/compiler/nuphar_handle.h" - -#include -#include - -namespace onnxruntime { -namespace nuphar { - -// Nuphar Tensor Context -struct TVMTensorCtx { - std::map inputs; - std::map> ops; - std::map> input_from; - - bool Lookup(const NodeArg* def, tvm::Tensor& tensor) { - const std::string& def_name = def->Name(); - auto iter = inputs.find(def_name); - if (iter != inputs.end()) { - tensor = iter->second; - return true; - } - - auto iter_out_index = input_from.find(def_name); - - if (iter_out_index == input_from.end()) { - return false; - } - - const Node* from_node = iter_out_index->second.first; - size_t index = iter_out_index->second.second; - auto iter_op = ops.find(from_node); - ORT_ENFORCE(iter_op != ops.end()); - tensor = iter_op->second[index]; - return true; - } - - const tvm::Tensor - Lookup(const NodeArg* def) const { - const std::string& def_name = def->Name(); - auto iter = inputs.find(def_name); - if (iter != inputs.end()) { - return iter->second; - } - - auto iter_out_index = input_from.find(def_name); - - ORT_ENFORCE(iter_out_index != input_from.end()); - - const Node* from_node = iter_out_index->second.first; - size_t index = iter_out_index->second.second; - auto iter_op = ops.find(from_node); - ORT_ENFORCE(iter_op != ops.end()); - return iter_op->second[index]; - } -}; - -struct WeightLayoutCtx { - //std::map initializer_to_weight_layout; // unused yet. This is for decoupling weight layout compile and run - std::unordered_map weight_layout_to_packed_func; -}; - -// NupharCodeGenCtx is Nuphar-specific CodeGenContext -class NupharCodeGenCtx : public tvm_codegen::CodeGenContext { - public: - NupharCodeGenCtx(const Node& node, - const std::map& initializers, - std::unordered_map>& global_generated_initializers, - const NupharCodeGenHandle* handle); - - NupharCodeGenCtx(const nuphar::NupharSubgraphUnit& subgraph, - std::unordered_map>& global_generated_initializers, - const NupharCodeGenHandle* handle); - - virtual ~NupharCodeGenCtx() = default; - - const NupharSubgraphUnitStats* GetGraphStats() const; - - bool IsInitializer(const std::string& name) const; - const Tensor* GetOrtInitializerTensor(const std::string& name) const; - WeightLayoutCodegenInfo* GetWeightLayoutInfo(const std::string& name); - const WeightLayoutCodegenInfo* GetWeightLayoutInfo(const std::string& name) const; - void CreateWeightLayoutInfo(const std::string& name, const tvm::Tensor& tensor); - const std::map>& GetWeightLayoutMap() const; - - // On-the-fly apply an existing layout - tvm::Tensor ApplyWeightLayout( - const std::string& layout_key, - const std::string& initializer_name, - const tvm::Tensor& X, - bool returnMarshalled); - - void RecordTensorToNode(const tvm::Tensor& t, const Node* node); - const Node* FindNode(const tvm::Tensor& t) const; - - const NupharCodeGenHandle* GetCodeGenHandle() const; - - // TODO remove this after decoupling compiler and runtime of WeightLayout - template - IAllocatorUniquePtr AllocateT(size_t size) const { return IAllocator::MakeUniquePtr(nuphar_handle_->allocator, size); } - // TODO remove this after decoupling compiler and runtime of WeightLayout - IAllocatorUniquePtr Allocate(size_t size) const { return AllocateT(size); } - - // Keep for CodeGenContext - TVMTensorCtx& GetTVMTensorCtx() { - return tvm_tensor_ctx_; - } - - // Keep for CodeGenContext - const TVMTensorCtx& GetTVMTensorCtx() const { - return tvm_tensor_ctx_; - } - - void InsertLiteral(const std::string& str) { - literalized_scalars_.insert(str); - } - - bool CheckLiteral(const std::string& str) { - return literalized_scalars_.count(str) > 0; - } - - private: - std::set literalized_scalars_; - - std::unique_ptr graph_stats_; - - const NupharCodeGenHandle* nuphar_handle_; - - const std::map& initializers_; - - // A table from tvm::Tensor (its unchanged source tvm::Node*) to ORT Node - std::unordered_map tvm_tensor_to_node_lookup_; - - // All TVM Tensor and correponidng shape context - TVMTensorCtx tvm_tensor_ctx_; - - // local copy - std::map> initializer_layouts_; - - std::unordered_map>& global_generated_initializers_; - - // all layouts - WeightLayoutCtx weight_layout_ctx_; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc deleted file mode 100644 index 4b648e0e38..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.cc +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/nuphar_compiler.h" - -#include "core/codegen/common/profile.h" -#include "core/codegen/common/settings.h" -#include "core/codegen/mti/mti_tvm_utils.h" -#include "core/codegen/passes/utils/ort_tvm_utils.h" -#include "core/mlas/inc/mlas.h" -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" -#include "core/providers/nuphar/common/nuphar_settings.h" -#include "core/providers/nuphar/common/nuphar_tvm_utils.h" -#include "core/providers/nuphar/compiler/nuphar_handle.h" -#include "core/providers/nuphar/compiler/nuphar_op_ir_builder.h" -#include "core/providers/nuphar/compiler/nuphar_schedule_builder.h" - -namespace onnxruntime { -namespace nuphar { - -static void HandleAllOutputs( - const std::vector& outputs, - tvm::Array& tvm_args, - tvm::Array& tvm_outputs, - const NupharCodeGenCtx& context) { - // find out all outputs - std::set visited_alias_def; - auto add_tvm_arg_and_output = [&](const onnxruntime::NodeArg* def) { - auto& tvm_tensor = context.GetTVMTensorCtx().Lookup(def); - tvm_args.push_back(tvm_tensor); - tvm_outputs.push_back(tvm_tensor); - }; - - for (const NodeArg* def : outputs) { - const NodeArg* input_def = Promote(context.GetGraphStats())->SourceDefOfOutputAlias(def); - if (input_def) { - auto key = GetKey(input_def); - if (visited_alias_def.count(key) == 0) { - visited_alias_def.insert(key); - add_tvm_arg_and_output(input_def); - } - } else { - auto key = GetKey(def); - if (visited_alias_def.count(key) == 0) { - visited_alias_def.insert(key); - add_tvm_arg_and_output(def); - } - } - } -} - -// Constructor for Node -// This is mainly for single node support -// For multiple subgraph support, please call the next constructor -NupharCompiler::NupharCompiler(const Node& node, - const std::map& initializer, - std::unordered_map>& generated_initializers, - const NupharCodeGenHandle* handle) - : num_initializers_in_graph_inputs_(0), - context_(node, initializer, generated_initializers, handle) {} - -NupharCompiler::NupharCompiler(const nuphar::NupharSubgraphUnit& subgraph, - std::unordered_map>& generated_initializers, - const NupharCodeGenHandle* handle) - : num_initializers_in_graph_inputs_(0), - context_(subgraph, generated_initializers, handle) {} - -Status NupharCompiler::Build(const nuphar::NupharSubgraphUnit& subgraph) { - if (subgraph.nodes.front()->OpType() == "Scan") { - return BuildSubgraph(*subgraph.nodes.front()); - } - - tvm_args_ = tvm::Array(); - tvm_outputs_ = tvm::Array(); - - ORT_RETURN_IF_ERROR(CreateTVMIR(subgraph, context_)); - - // fill in all non-initializer inputs - num_initializers_in_graph_inputs_ = 0; - for (auto& def : subgraph.inputs) { - if (context_.IsInitializer(def->Name())) { - ++num_initializers_in_graph_inputs_; - } else { - tvm_args_.push_back(context_.GetTVMTensorCtx().Lookup(def)); - } - } - - // fill in all initializers - for (const auto& item : context_.GetWeightLayoutMap()) { - const WeightLayoutCodegenInfo* layout_info = item.second.get(); - tvm_args_.push_back(layout_info->marshalled_tensor); - } - - // find out all outputs, and save the output shapes - HandleAllOutputs(subgraph.outputs, tvm_args_, tvm_outputs_, context_); - - return Status::OK(); -} - -// BuildSubgraph drive a graph traversal that calls CreateInput and CreateOutputs metioned above for a subgraph. -// And collect args among nodes. -// We need another API other than Build, because name mismatching -Status NupharCompiler::BuildSubgraph(const Node& node) { - tvm_args_ = tvm::Array(); - tvm_outputs_ = tvm::Array(); - - auto subgraph = GetSubgraph(node); - - ORT_RETURN_IF_ERROR(CreateTVMIR(GraphViewer(*subgraph), context_, /*use_placeholder_for_input*/ true)); - - num_initializers_in_graph_inputs_ = 0; - // fill in all non-initializer inputs - - for (const auto& input : subgraph->GetInputs()) { - if (context_.IsInitializer(input->Name())) { - ++num_initializers_in_graph_inputs_; - } else { - tvm_args_.push_back(context_.GetTVMTensorCtx().Lookup(input)); - } - } - - // fill in implicit inputs - for (const auto& input : node.ImplicitInputDefs()) { - if (context_.IsInitializer(input->Name())) { - ++num_initializers_in_graph_inputs_; - } else { - tvm_args_.push_back(context_.GetTVMTensorCtx().Lookup(input)); - } - } - - // fill in all initializers - for (const auto& item : context_.GetWeightLayoutMap()) { - const WeightLayoutCodegenInfo* layout_info = item.second.get(); - tvm_args_.push_back(layout_info->marshalled_tensor); - } - - // find out all outputs - HandleAllOutputs(subgraph->GetOutputs(), tvm_args_, tvm_outputs_, context_); - - return Status::OK(); -} - -tvm::runtime::PackedFunc NupharCompiler::GetLoweredPackedFunc( - const std::string& func_name, - tvm::Target tvm_target, - tvm::Target tvm_host_target, - const tvm::BuildConfig& config, - const std::string& subgraph_type, - const std::string& subgraph_name) { - // TODO: refactor the following logic for both JIT-caching and AOT support - // JIT-caching and AOT are mutual exclusive. - // Change it by not always saving a compiled func unless it is in JIT-Caching model. - // In AOT, there should be another member func explicitly loading - tvm::runtime::PackedFunc cached_func; - auto cache_status = nuphar::LoadTVMPackedFuncFromCache(func_name, cached_func); - if (cache_status != nuphar::CacheStatus::Found) { - codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - - if (settings.HasOption(kNupharCacheForceNoJIT)) { - if (settings.OptionMatches(kNupharCacheForceNoJIT, "on")) { - ORT_THROW("Force not using JIT code!"); - } - } - - tvm::Schedule tvm_schedule = CreateSchedule(tvm_outputs_, context_); - std::unordered_map binds; - tvm::Array lowered = tvm::lower(tvm_schedule, tvm_args_, func_name, binds, config); - - if (settings.HasOption(codegen::CodeGenSettings::kCodeGenDumpLower)) { - if (settings.OptionMatches(codegen::CodeGenSettings::kCodeGenDumpLower, "verbose") || - settings.OptionMatches(codegen::CodeGenSettings::kCodeGenDumpLower, subgraph_type)) { - for (const auto& func : lowered) - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "[CODEGEN_DUMP_LOWER] Dumping lowered func: " << func << std::endl - << func->body; - } else if (settings.OptionMatches(codegen::CodeGenSettings::kCodeGenDumpLower, "concise")) { - LOGS_DEFAULT(CODEGEN_SETTINGS_LOG_LEVEL) << "[CODEGEN_DUMP_LOWER] Subgraph Type: " - << subgraph_type << ", name: " << subgraph_name - << " #lowered funcs: " << lowered.size() << std::endl; - } - } - - tvm::runtime::Module module = tvm::build(lowered, tvm_target, tvm_host_target, config); - tvm_codegen::DumpTVMModuleToFile(func_name, module); - if (cache_status == nuphar::CacheStatus::Missing) { - nuphar::SaveTVMModuleToCache(func_name, module); - } - cached_func = module.GetFunction(func_name); - } - - return cached_func; -} - -static tvm::BuildConfig CreateConfig(const Node& node, - bool allow_unaligned_buffers) { - tvm::BuildConfig config = tvm::build_config(); - config->disable_select_rewriting = true; - - if (allow_unaligned_buffers) { - config->data_alignment = 1; // aligned to 1 - } else { - config->data_alignment = gsl::narrow(MlasGetPreferredBufferAlignment()); - } - - config->restricted_func = true; - return config; -} - -// Lower compiles the tvm::Tensor to a function -Status NupharCompiler::Lower(const nuphar::NupharSubgraphUnit& subgraph, - tvm::Target tvm_target, - tvm::Target tvm_host_target, - NupharFuncInfo* func_info, - nuphar::OrtSubgraphAllocationInfo* partition_info) { - const auto& codegen_handle = context_.GetCodeGenHandle(); - const auto& target_codegen = *codegen_handle->codegen_target; - std::string func_name = nuphar::GetPackedFuncName(subgraph, target_codegen, codegen_handle->parallel_min_workloads); - tvm::BuildConfig config = CreateConfig(*subgraph.nodes.front(), - context_.GetCodeGenHandle()->allow_unaligned_buffers); - - // using "subgraph" for type and name for now - // TODO: change name - tvm::runtime::PackedFunc cached_func = - GetLoweredPackedFunc( - func_name, tvm_target, tvm_host_target, - config, "subgraph", "subgraph"); - - FillNupharFuncInfo(func_info, partition_info, subgraph, context_, tvm_target, cached_func, func_name); - - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.h b/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.h deleted file mode 100644 index f6d0f7ca17..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_compiler.h +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/codegen/common/common.h" -#include "core/providers/nuphar/common/nuphar_subgraph.h" -#include "core/providers/nuphar/compiler/func_info.h" -#include "core/providers/nuphar/compiler/initializer_info.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/compiler/nuphar_handle.h" -#include "core/providers/nuphar/compiler/traverse_shape_infer.h" -#include "core/framework/op_kernel.h" -#include "core/graph/graph.h" -#include "gsl/gsl" - -#include -#include - -namespace onnxruntime { -namespace nuphar { - -class NupharCompiler { - public: - NupharCompiler(const Node& node, - const std::map& initializers, - std::unordered_map>& generated_initializers, - const NupharCodeGenHandle* handle); - - NupharCompiler(const nuphar::NupharSubgraphUnit& subgraph, - std::unordered_map>& generated_initializers, - const NupharCodeGenHandle* handle); - - // Build builds tvm IR and apply passes - Status Build(const nuphar::NupharSubgraphUnit& subgraph); - - // Lower lowers the built tvm IR to llvm ir and compiles it - Status Lower(const nuphar::NupharSubgraphUnit& subgraph, - tvm::Target tvm_target, - tvm::Target tvm_host_target, - NupharFuncInfo* ctx_func, - nuphar::OrtSubgraphAllocationInfo* partition_info); - - tvm::runtime::PackedFunc GetLoweredPackedFunc( - const std::string& func_name, - tvm::Target tvm_target, - tvm::Target tvm_host_target, - const tvm::BuildConfig& config, - const std::string& subgraph_type, - const std::string& subgraph_name); - - private: - size_t num_initializers_in_graph_inputs_; - - // BuildSubgraph builds tvm IR and apply passes for a subgraph - Status BuildSubgraph(const Node& node); - - NupharCodeGenCtx context_; - - tvm::Array tvm_args_; - tvm::Array tvm_outputs_; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_handle.h b/onnxruntime/core/providers/nuphar/compiler/nuphar_handle.h deleted file mode 100644 index 1b5f91990d..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_handle.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/codegen/common/common.h" -#include "core/codegen/common/handle.h" -#include "core/codegen/common/target_info.h" -#include "core/codegen/passes/weight_layout/weight_layout.h" -#include "core/framework/allocator.h" // TODO: get rid of this -#include "core/providers/nuphar/compiler/traverse_shape_infer.h" // TODO: get rid of this - -namespace onnxruntime { - -// forwarding -namespace tvm_codegen { -class TVMIRBuilder; -class TVMScheduleBuilder; -} // namespace tvm_codegen - -namespace nuphar { - -// TVM is a wrapper containing CodeGen related setting -// TODO: make this the Base -// TODO: create one for nuphar -struct NupharCodeGenHandle : codegen::CodeGenHandle { - std::shared_ptr op_ir_builder; // keep - std::shared_ptr schedule_builder; // keep - // maybe add a layout - tvm_codegen::WeightLayoutRegistry* layout_registry; - int64_t parallel_min_workloads; - - bool allow_unaligned_buffers; // move to another place - - AllocatorPtr allocator; // remove - std::shared_ptr shape_inference; // remove -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc deleted file mode 100644 index 5c4540dba6..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.cc +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/nuphar_op_ir_builder.h" - -#include "core/codegen/mti/mti_tvm_utils.h" -#include "core/codegen/passes/op_ir_creator/all_ops.h" -#include "core/codegen/passes/op_ir_creator/tvm_ir_builder.h" -#include "core/codegen/passes/utils/ort_tvm_utils.h" -#include "core/common/common.h" -#include "core/providers/nuphar/common/nuphar_tvm_utils.h" -#include "core/providers/nuphar/compiler/initializer_info.h" -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -namespace onnxruntime { -namespace nuphar { - -// Declaration of GetOrCreateInitializer -// GetOrCreateInitializer create tvm::placeholder for a marshalled weight -// with correpsonding data layout transfomration for a weight, -// Note the weight is fed during build -static const tvm::Tensor& GetOrCreateInitializer(const std::string& name, - const Tensor* tensor, - bool is_sliced, - NupharCodeGenCtx& ctx_codegen); - -static const tvm::Tensor& GetOrCreateInitializer(const NodeArg* def, - const Tensor* tensor, - bool is_sliced, - NupharCodeGenCtx& ctx_codegen); - -static bool CreateScalarTensorFromInitializer(const Tensor* tensor, - const std::string& name, - NupharCodeGenCtx& ctx_codegen); - -// CreateInputPlaceholder create tvm input placeholder (tvm::Tensor) -// NOTE: here we assume axis 0 is sequence -// TODO: add support for sequence not axis 0 -static tvm::Tensor CreateInputPlaceholder(const tvm::Array& shape, - HalideIR::Type halide_type, - const std::string& name, - bool is_sliced) { - return tvm::placeholder(is_sliced && shape.size() > 1 ? tvm_codegen::SliceShapeFromDimension(shape, 1) : shape, halide_type, name); -} - -// CreateInput creats tvm::Tensor of corresponding ORT input -// Inputs are either initializer or regular input placeholder -static bool CreateInput( - const NodeArg* def, - tvm::Tensor& input, - bool initializer_only, - bool is_sliced, - NupharCodeGenCtx& ctx_codegen) { - const Tensor* initialized_tensor = ctx_codegen.GetOrtInitializerTensor(def->Name()); - if (nullptr == initialized_tensor && initializer_only) - return false; - - ORT_ENFORCE(def->Shape()); - - if (nullptr != initialized_tensor && - CreateScalarTensorFromInitializer(initialized_tensor, def->Name(), ctx_codegen)) { - return false; // constant scalar tensor do not need to be in input - } - - if (nullptr != initialized_tensor) { - input = GetOrCreateInitializer(def, initialized_tensor, is_sliced, ctx_codegen); - } else { - // Handle inputs without initializer - std::string name = NormalizeNodeArgName(def); - MLDataType ONNXRUNTIME_data_type = DataTypeImpl::TypeFromProto(*def->TypeAsProto()); - DLDataType dtype = tvm_codegen::ToTvmDLDataType(ONNXRUNTIME_data_type); - HalideIR::Type halide_type((halideir_type_code_t)dtype.code, dtype.bits, dtype.lanes); - tvm::Array shape = ShapeToTvmArray(def, ctx_codegen); - - // Create InputPlaceholder - // Slice InputPlaceholder if it is asked for. - input = CreateInputPlaceholder(shape, halide_type, name, is_sliced); - } - return true; -} - -bool CreateScalarTensorFromInitializer(const Tensor* tensor, - const std::string& name, - NupharCodeGenCtx& ctx_codegen) { - TVMTensorCtx& ctx_tensor = ctx_codegen.GetTVMTensorCtx(); - ORT_ENFORCE(tensor != nullptr); - - tvm::Expr constant_scalar; - if (!TryCreateConstantScalar(constant_scalar, tensor)) - return false; - - std::string normalized_name = NormalizeCppName(name); - auto tvm_tensor = tvm::compute( - tvm_codegen::ToTvmArray(tensor->Shape().GetDims()), - [&](const tvm::Array&) { - return constant_scalar; - }, - normalized_name); - - ctx_codegen.InsertLiteral(normalized_name); - ctx_tensor.inputs.emplace(name, std::move(tvm_tensor)); - return true; -} - -// GetOrCreateInitializer create tvm::placeholder for a marshalled weight -// with correpsonding data layout transfomration for a weight, -// Note the weight is fed during build -const tvm::Tensor& GetOrCreateInitializer(const std::string& name, - const Tensor* tensor, - bool is_sliced, - NupharCodeGenCtx& ctx_codegen) { - ORT_ENFORCE(ctx_codegen.IsInitializer(name)); - - auto layout_info = ctx_codegen.GetWeightLayoutInfo(name); - if (nullptr != layout_info) { - return layout_info->marshalled_tensor; - } - - auto ONNXRUNTIME_data_type = tensor->DataType(); - DLDataType dtype = tvm_codegen::ToTvmDLDataType(ONNXRUNTIME_data_type); - HalideIR::Type halide_type((halideir_type_code_t)dtype.code, dtype.bits, dtype.lanes); - std::string normalized_name = NormalizeCppName(name); - auto tvm_shape = tvm_codegen::ToTvmArray(tensor->Shape().GetDims()); - auto tvm_tensor = CreateInputPlaceholder(tvm_shape, halide_type, normalized_name, is_sliced); - // create the layout info - ctx_codegen.CreateWeightLayoutInfo(name, tvm_tensor); - return ctx_codegen.GetWeightLayoutInfo(name)->marshalled_tensor; -} - -const tvm::Tensor& GetOrCreateInitializer(const NodeArg* def, - const Tensor* tensor, - bool is_sliced, - NupharCodeGenCtx& ctx_codegen) { - return GetOrCreateInitializer(def->Name(), tensor, is_sliced, ctx_codegen); -} - -// CreateOutputs constructs tvm::Tensor with corresponding computation -static Status CreateOutputs(const Node* node, - const tvm::Array& inputs, - tvm::Array& outputs, - NupharCodeGenCtx& ctx_codegen) { - ORT_RETURN_IF_ERROR(ctx_codegen.GetCodeGenHandle() - ->op_ir_builder - ->Evaluate(inputs, *node, ctx_codegen, outputs)); - - // Collect constructed tvm::Node to onnxruntime::Node mapping - // Both states and outputs - for (const auto& t : outputs) { - ctx_codegen.RecordTensorToNode(t, node); - } - - return Status::OK(); -} - -// CreateTVMIR is the entry function for building TVM IR -// It will call TVMIRBuilder (in CreateOutputs) from CodeGenContext -Status CreateTVMIR( - const GraphViewer& graph, - NupharCodeGenCtx& ctx_codegen, - bool use_placeholder_for_input) { - TVMTensorCtx& ctx_tensor = ctx_codegen.GetTVMTensorCtx(); - - if (use_placeholder_for_input) { - // build graph inputs - const auto& graph_inputs = graph.GetInputs(); - for (size_t i = 0; i < graph_inputs.size(); ++i) { - tvm::Tensor value; - if (CreateInput(graph_inputs[i], value, - /*initializer_only*/ false, /*is_sliced*/ false, - ctx_codegen)) { - ctx_tensor.inputs.emplace(graph_inputs[i]->Name(), std::move(value)); - } - } - } - - for (const auto& node : graph.Nodes()) { - // initializers - ORT_RETURN_IF_ERROR(node.ForEachWithIndex( - node.InputDefs(), - [&ctx_codegen, &ctx_tensor](const NodeArg& def, size_t) { - tvm::Tensor value; - if (CreateInput(&def, value, /*initializer_only*/ true, /*is_sliced*/ false, - ctx_codegen)) { - ctx_tensor.inputs.emplace(def.Name(), std::move(value)); - } - return Status::OK(); - })); - } - - // iterate through the graph and create op (outputs) - for (auto node_index : graph.GetNodesInTopologicalOrder()) { - const auto& node = *graph.GetNode(node_index); - tvm::Array inputs; - for (const NodeArg* def : node.InputDefs()) { - tvm::Tensor input; - if (def->Exists()) { - bool exist = ctx_tensor.Lookup(def, input); - if (!exist) { - tvm::Tensor value; - if (CreateInput(def, value, - /*initializer_only*/ false, /*is_sliced*/ false, - ctx_codegen)) { - ctx_tensor.inputs.emplace(def->Name(), std::move(value)); - } - input = ctx_tensor.Lookup(def); - } - } - inputs.push_back(input); - } - - auto subgraph = GetSubgraph(node); - if (nullptr != subgraph) { - // unboxing - GraphViewer subgraph_viewer(*subgraph); - ORT_RETURN_IF_ERROR(CreateTVMIR(subgraph_viewer, ctx_codegen, /*use_placeholder_for_input*/ false)); - } else { - tvm::Array op_outputs; - ORT_RETURN_IF_ERROR(CreateOutputs(&node, inputs, op_outputs, ctx_codegen)); - ctx_tensor.ops.emplace(&node, std::move(op_outputs)); - - // input_from_ - ORT_RETURN_IF_ERROR(node.ForEachWithIndex( - node.OutputDefs(), - [&node, &ctx_tensor](const NodeArg& def, size_t index) { - ORT_ENFORCE(ctx_tensor.input_from.count(def.Name()) == 0); - ctx_tensor.input_from.emplace(def.Name(), std::make_pair(&node, index)); - return Status::OK(); - })); - } - } - - return Status::OK(); -} - -// CreateTVMIR is the entry function for building TVM IR -// It will call TVMIRBuilder (in CreateOutputs) from CodeGenContext -Status CreateTVMIR( - const Node& node, - NupharCodeGenCtx& ctx_codegen) { - // wrapper - TVMTensorCtx& ctx_tensor = ctx_codegen.GetTVMTensorCtx(); - bool has_loop = HasLoop(node); - - // create real Inputs - ORT_RETURN_IF_ERROR(node.ForEachWithIndex( - node.InputDefs(), - [&has_loop, &ctx_codegen, &ctx_tensor](const NodeArg& def, size_t) { - tvm::Tensor value; - if (CreateInput(&def, value, /*initializer_only*/ false, /*is_sliced*/ has_loop, - ctx_codegen)) { - ctx_tensor.inputs.emplace(def.Name(), std::move(value)); - } - return Status::OK(); - })); - - // input_from_ - ORT_RETURN_IF_ERROR(node.ForEachWithIndex( - node.OutputDefs(), - [&node, &ctx_tensor](const NodeArg& def, size_t index) { - ctx_tensor.input_from.emplace(def.Name(), std::make_pair(&node, index)); - return Status::OK(); - })); - - tvm::Array inputs; - for (const NodeArg* def : node.InputDefs()) { - inputs.push_back(def->Exists() ? ctx_tensor.Lookup(def) : tvm::Tensor()); - } - - // create ops (outputs) - tvm::Array op_outputs; - ORT_RETURN_IF_ERROR(CreateOutputs(&node, inputs, op_outputs, ctx_codegen)); - ctx_tensor.ops.emplace(&node, std::move(op_outputs)); - - return Status::OK(); -} - -// CreateTVMIR is the entry function for building TVM IR -// It will call TVMIRBuilder (in CreateOutputs) from CodeGenContext -Status CreateTVMIR( - const nuphar::NupharSubgraphUnit& subgraph, - NupharCodeGenCtx& ctx_codegen) { - //////////////////////////////////////// - // handle a special case for a single node - //////////////////////////////////////// - if (subgraph.IsSingleNode()) { - const Node* node = subgraph.nodes.front(); - - const Graph* onnx_graph = GetSubgraph(*node); - - if (nullptr != onnx_graph) { - return CreateTVMIR(GraphViewer(*onnx_graph), ctx_codegen, true); - } - return CreateTVMIR(*node, ctx_codegen); - } - - ////////////////////////////// - // handle a generic subgraph below - ////////////////////////////// - TVMTensorCtx& ctx_tensor = ctx_codegen.GetTVMTensorCtx(); - - // build subgraph inputs - for (const NodeArg* def : subgraph.inputs) { - tvm::Tensor value; - - if (CreateInput(def, value, /*initializer_only*/ false, /*is_sliced*/ false, - ctx_codegen)) { - ctx_tensor.inputs.emplace(def->Name(), std::move(value)); - } - } - - // build subgraph initializers - for (auto& p : subgraph.initializers) { - tvm::Tensor value = GetOrCreateInitializer(p.first, p.second, false, ctx_codegen); - ctx_tensor.inputs.emplace(p.first, std::move(value)); - } - - // iterate through the subgraph nodes and create op (outputs) - for (auto& node : subgraph.nodes) { - tvm::Array inputs; - - // collects local inputs - for (const NodeArg* def : node->InputDefs()) { - inputs.push_back(def->Exists() ? ctx_tensor.Lookup(def) : tvm::Tensor()); - } - - tvm::Array op_outputs; - ORT_RETURN_IF_ERROR(CreateOutputs(node, inputs, op_outputs, ctx_codegen)); - ctx_tensor.ops.emplace(node, std::move(op_outputs)); - - // input_from_ - ORT_RETURN_IF_ERROR(node->ForEachWithIndex( - node->OutputDefs(), - [&node, &ctx_tensor](const NodeArg& def, size_t index) { - ORT_ENFORCE(ctx_tensor.input_from.count(def.Name()) == 0); - ctx_tensor.input_from.emplace(def.Name(), std::make_pair(node, index)); - return Status::OK(); - })); - } - - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.h b/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.h deleted file mode 100644 index 532e917b5d..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_op_ir_builder.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/common/common.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" - -#include "core/providers/nuphar/common/nuphar_subgraph.h" - -namespace onnxruntime { -namespace nuphar { - -// CreateTVMIR function traverses a GraphViewer -// and builds tvm ir (and store them in CodeGenContext) -// based on corresponding ORT ir -Status CreateTVMIR(const GraphViewer& graph, - NupharCodeGenCtx& ctx_codegen, - bool use_placeholder_for_input); - -// CreateTVMIR function traverses a single node -// and builds tvm ir (and store them in CodeGenContext) -// based on corresponding ORT ir -Status CreateTVMIR(const Node& node, - NupharCodeGenCtx& ctx_codegen); - -// CreateTVMIR function traverses a NupharSubgraphUnit -// and builds tvm ir (and store them in CodeGenContext) -// based on corresponding ORT ir -Status CreateTVMIR(const nuphar::NupharSubgraphUnit& subgraph, - NupharCodeGenCtx& ctx_codegen); - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_schedule_builder.cc b/onnxruntime/core/providers/nuphar/compiler/nuphar_schedule_builder.cc deleted file mode 100644 index c05848a04c..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_schedule_builder.cc +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/nuphar_schedule_builder.h" - -#include "core/codegen/common/settings.h" -#include "core/codegen/passes/scheduler/schedule_utils.h" -#include "core/codegen/passes/scheduler/tvm_schedule_builder.h" - -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" -#include "core/providers/nuphar/compiler/x86/x86_target_info.h" -#include "core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h" - -// TODO change name space -namespace onnxruntime { -namespace nuphar { - -// Traverse iterates a tvm::Tensor and itself dependencies -// and builds schedule (in ScheduleContext) -// based on corresponding ORT ir and TVM ir -static void Traverse(const tvm::Tensor& tensor, - const Node* node, - NupharCodeGenCtx& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_schedule) { - // no need to traverse on nodes already marked as closured - if (ctx_schedule.scheduled_tensors.count(tensor->op.get()) > 0) { - if (ctx_schedule.scheduled_tensors[tensor->op.get()] == tvm_codegen::ScheduleType::ScheduleClosure) { - return; - } - } - - ORT_THROW_IF_ERROR(ctx_codegen.GetCodeGenHandle()->schedule_builder->Evaluate(tensor, node, ctx_codegen, ctx_schedule)); - - // for real ouput - bool is_real_output = nullptr != node && - Promote(ctx_codegen.GetGraphStats())->IsOutputNode(node); - - if (is_real_output) { - CodeGenTargetX86* target = dynamic_cast(ctx_codegen.GetCodeGenHandle()->codegen_target); - ORT_ENFORCE(target != nullptr); - - TryVectorizationX86(tensor, ctx_codegen, ctx_schedule); - InsertRootScheduleAndClosure(tensor, ctx_schedule); - } - - // Traverse tensor's children - for (auto& t : tensor->op->InputTensors()) { - // check whether it is a tensor having inputs - if (t->op->InputTensors().size() > 0) { - auto current_node = ctx_codegen.FindNode(t); - Traverse(t, current_node, ctx_codegen, ctx_schedule); - } else if (ctx_codegen.CheckLiteral(t->op->name)) { - if (tensor->op.as() != nullptr) { - InsertRootSchedule(t, ctx_schedule); - } else { - TryInlineSchedule(t, ctx_schedule); - } - } - } -} - -tvm::Schedule CreateSchedule(const tvm::Array& outs, - NupharCodeGenCtx& ctx_codegen) { - // Create scheudule object - tvm::Array out_ops; - for (auto& t : outs) { - out_ops.push_back(t->op); - } - - if (codegen::CodeGenSettings::Instance().HasOption(codegen::CodeGenSettings::kCodeGenDumpSchedule)) - ctx_codegen.GetCodeGenHandle()->schedule_builder->DumpAllSchedulers(); - - tvm_codegen::ScheduleContext ctx_schedule(out_ops); - - // Schedule all outputs - for (const auto& t : outs) { - const Node* node = ctx_codegen.FindNode(t); - Traverse(t, node, ctx_codegen, ctx_schedule); - } - - return ctx_schedule.schedule; -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/nuphar_schedule_builder.h b/onnxruntime/core/providers/nuphar/compiler/nuphar_schedule_builder.h deleted file mode 100644 index de4631b154..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/nuphar_schedule_builder.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include -#include "core/common/common.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" - -// TODO change name space -namespace onnxruntime { -namespace nuphar { - -// Traverse iterates tvm::Array a single node -// and builds the whole schedule (in CodeGenContext) -tvm::Schedule CreateSchedule(const tvm::Array& outs, - NupharCodeGenCtx& ctx_codegen); - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/traverse_shape_infer.cc b/onnxruntime/core/providers/nuphar/compiler/traverse_shape_infer.cc deleted file mode 100644 index 659ddc3804..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/traverse_shape_infer.cc +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/traverse_shape_infer.h" - -#include "core/codegen/common/common.h" -#include "core/common/common.h" -#include "core/framework/tensorprotoutils.h" - -// TODO retire this file - -namespace onnxruntime { -namespace nuphar { - -// local shape infernece function for input -static bool CreateInput(const NodeArg* def, - const GraphViewer& graph, - ShapeExpr& input, - bool initializer_only) { - if (initializer_only && graph.GetAllInitializedTensors().count(def->Name()) == 0) - return false; - - auto def_shape = def->Shape(); - if (!def_shape) - return false; - - int rank = def_shape->dim_size(); - input = ShapeExpr(rank); - for (int i = 0; i < rank; ++i) { - const auto& dim = def_shape->dim()[i]; - if (utils::HasDimValue(dim)) - input[i] = DimExpr(dim.dim_value()); - else if (utils::HasDimParam(dim)) - input[i] = DimExpr(dim.dim_param()); - else { - input[i] = DimExpr(NormalizeNodeArgName(def) + "_dim" + std::to_string(i)); - } - } - return true; -} - -// local shape infernece function for output -static Status CreateOutputs( - const Node* node, - const std::vector& inputs, - std::vector& outputs) { - outputs.resize(node->OutputDefs().size()); - ORT_RETURN_IF_ERROR(node->ForEachWithIndex( - node->OutputDefs(), - [&](const NodeArg& def, size_t index) { - auto shape_proto = def.Shape(); - if (shape_proto) { - TensorShape shape{utils::GetTensorShapeFromTensorShapeProto(*shape_proto)}; - ShapeExpr output_shape(shape.NumDimensions()); - for (int d = 0; d < gsl::narrow(shape.NumDimensions()); ++d) { - if (shape[d] > 0) { - output_shape[d] = DimExpr(shape[d]); - } else { - ORT_RETURN_IF_NOT(shape_proto->dim_size() > d && utils::HasDimParam(shape_proto->dim(d)), - "shape_proto->dim_size() > d && utils::HasDimParam(shape_proto->dim(d) was false"); - output_shape[d] = DimExpr(shape_proto->dim(d).dim_param()); - } - } - outputs[index] = output_shape; - } - return Status::OK(); - })); - return Status::OK(); -} - -// The main function for shape infernece -Status ShapeInference( - const GraphViewer& graph, - ShapeExprContext& context) { - // build graph inputs - const auto& graph_inputs = graph.GetInputs(); - for (size_t i = 0; i < graph_inputs.size(); ++i) { - ShapeExpr value; - if (CreateInput(graph_inputs[i], graph, value, /*initializer_only*/ false)) { - context.inputs.emplace(graph_inputs[i]->Name(), std::move(value)); - } - } - - // perform shape inference using the topological order from ORT - for (const NodeIndex& node_index : graph.GetNodesInTopologicalOrder()) { - const Node* p_node = graph.GetNode(node_index); - if(p_node == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "invalid node index"); - const Node& node = *p_node; - // initializers - ORT_RETURN_IF_ERROR(node.ForEachWithIndex( - node.InputDefs(), - [&graph, &context](const NodeArg& def, size_t) { - ShapeExpr value; - if (CreateInput(&def, graph, value, /*initializer_only*/ true)) { - context.inputs.emplace(def.Name(), std::move(value)); - } - return Status::OK(); - })); - - // handle subgraph - const Graph* subgraph = GetSubgraph(node); - if (nullptr != subgraph) { - GraphViewer subgraph_viewer(*subgraph); - ORT_RETURN_IF_ERROR(ShapeInference(subgraph_viewer, context)); - } - - // collect inputs before creating outputs - std::vector inputs; - for (const NodeArg* def : node.InputDefs()) { - inputs.push_back(def->Exists() ? context.Lookup(def) : nullptr); - } - - // create outputs - std::vector op_outputs; - ORT_RETURN_IF_ERROR(CreateOutputs(&node, inputs, op_outputs)); - context.ops.emplace(&node, std::move(op_outputs)); - - // recall input_from_ - ORT_RETURN_IF_ERROR(node.ForEachWithIndex( - node.OutputDefs(), - [&node, &context](const NodeArg& def, size_t index) { - context.input_from.emplace(def.Name(), std::make_pair(&node, index)); - return Status::OK(); - })); - } - - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/traverse_shape_infer.h b/onnxruntime/core/providers/nuphar/compiler/traverse_shape_infer.h deleted file mode 100644 index deaa5777a3..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/traverse_shape_infer.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/providers/nuphar/common/analysis/shape_expr.h" -#include "core/common/common.h" -#include "core/framework/tensor.h" -#include "core/graph/graph_viewer.h" - -namespace onnxruntime { -namespace nuphar { - -// A collection of ShapeExpr -struct ShapeExprContext { - std::map inputs; - std::map> ops; - std::map> input_from; - - const ShapeExpr* Lookup(const NodeArg* def) const { - const std::string& def_name = def->Name(); - auto iter = inputs.find(def_name); - if (iter != inputs.end()) - return &(iter->second); - - auto iter_out_index = input_from.find(def_name); - - // OK if shape inference is incomplete - // This is for some per-node unit test where NodeArg does not even have shape ranks - // We ignore the shape inference in ToCapacity computation in per-node unit tests - if (iter_out_index == input_from.end()) - return nullptr; - - const Node* from_node = iter_out_index->second.first; - size_t index = iter_out_index->second.second; - auto iter_op = ops.find(from_node); - ORT_ENFORCE(iter_op != ops.end()); - return &(iter_op->second[index]); - } -}; - -// Traverse function traverses a GraphViewer, -// performs shape infernce, -// and builds ShapeExpr in ShapeExprContext -Status ShapeInference(const GraphViewer& graph, - ShapeExprContext& context); - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h deleted file mode 100644 index b4ee871611..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/passes/utils/codegen_context.h" -#include "core/codegen/passes/op_ir_creator/tvm_op_creator.h" - -namespace onnxruntime { -namespace nuphar { - -// Declare a TVM IR builder based on the ORT OP type -// with postfix NupharTVMX86 -#define DECLARE_NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(OP) \ - DECLARE_OP_IR_CREATOR_CLASS_EX(OP, NupharTVM, X86) - -// Return a TVM IR builder class name such as OP type -// with postfix NupharTVMX86 -#define NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(OP) \ - OP_IR_CREATOR_CLASS_EX(OP, NupharTVM, X86) - -#define NUPHAR_TVM_X86_OP_IR_CREATOR_STRING(OP) \ - STRINGIZE(NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(OP)) - -#define LIST_X86_POOL_OPS() \ - POOL_OP(MaxPool) \ - POOL_OP(AveragePool) \ - POOL_OP(GlobalMaxPool) \ - POOL_OP(GlobalAveragePool) - -#define LIST_X86_UNARY_OPS() \ - UNARY_OP(Erf) \ - UNARY_OP(Exp) \ - UNARY_OP(Log) \ - UNARY_OP(ParametricSoftplus) \ - UNARY_OP(ScaledTanh) \ - UNARY_OP(Selu) \ - UNARY_OP(Sigmoid) \ - UNARY_OP(Softplus) \ - UNARY_OP(Tanh) - -#define LIST_REDUCE_V_OPS() \ - REDUCE_V_OP(ReduceMax) \ - REDUCE_V_OP(ReduceMin) \ - REDUCE_V_OP(ReduceSum) \ - REDUCE_V_OP(ReduceMean) - -#define LIST_ALL_X86_OPS() \ - LIST_REDUCE_V_OPS() \ - LIST_X86_POOL_OPS() \ - LIST_X86_UNARY_OPS() \ - ADD_OP_ITEM(Dropout) \ - ADD_OP_ITEM(Gemm) \ - ADD_OP_ITEM(LogSoftmax) \ - ADD_OP_ITEM(MatMul) \ - ADD_OP_ITEM(MatMulInteger) \ - ADD_OP_ITEM(MatMulInteger16) \ - ADD_OP_ITEM(Pow) \ - ADD_OP_ITEM(Scatter) \ - ADD_OP_ITEM(ScatterElements) \ - ADD_OP_ITEM(Slice) \ - ADD_OP_ITEM(Softmax) \ - ADD_OP_ITEM(Tile) - -// Define all OPs for NupharTVMX86 -#define ADD_OP_ITEM(OP) DECLARE_NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(OP) -#define POOL_OP(OP) ADD_OP_ITEM(OP) -#define REDUCE_V_OP(OP) ADD_OP_ITEM(OP) -#define UNARY_OP(OP) ADD_OP_ITEM(OP) - -LIST_ALL_X86_OPS() - -#undef ADD_OP_ITEM -#undef REDUCE_V_OP -#undef POOL_OP -#undef UNARY_OP - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/gemm.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/gemm.cc deleted file mode 100644 index 89cd5857b4..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/gemm.cc +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -#include "core/codegen/mti/math/binary_ops.h" -#include "core/codegen/mti/math/gemm.h" -#include "core/codegen/mti/mti_tvm_utils.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/mti_x86/math/matmul_ops.h" - -namespace onnxruntime { -namespace nuphar { - -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Gemm)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - - tvm::Tensor Y; - auto& A = inputs[0]; - auto& B = inputs[1]; - tvm::Tensor C; - - int64_t trans_a, trans_b; - float alpha, beta; - ORT_RETURN_IF_ERROR(info.GetAttr("transA", &trans_a)); - ORT_RETURN_IF_ERROR(info.GetAttr("transB", &trans_b)); - ORT_RETURN_IF_ERROR(info.GetAttr("alpha", &alpha)); - ORT_RETURN_IF_ERROR(info.GetAttr("beta", &beta)); - - // bias is optional - if (inputs.size() < 3) { - beta = 0; - C = tvm_codegen::MakeZeroTensor({1}, A->dtype, node.Name() + "_zero"); - } else { - C = inputs[2]; - } - - // use native sgemm for floating point - if (A->dtype == HalideIR::Float(32) && - B->dtype == HalideIR::Float(32) && - GemmExternCpu(A, B, Y, !!trans_a, !!trans_b, node.Name() + "_gemm")) { - if (beta != 0) { - tvm::Tensor beta_bias = (beta == 1) ? C : tvm_codegen::Mul(tvm::make_const(tvm::Float(32), beta), C); - Y = tvm_codegen::Add((alpha == 1) ? Y : tvm_codegen::Mul(tvm::make_const(tvm::Float(32), alpha), Y), - beta_bias, node.Name() + "_add_bias"); - } else { - Y = (alpha == 1) ? Y : tvm_codegen::Mul(tvm::make_const(tvm::Float(32), alpha), Y); - } - outputs.push_back(Y); - return Status::OK(); - } - - // fallback to default MTI ops - Y = tvm_codegen::Gemm(A, B, C, trans_a, trans_b, alpha, beta, node.Name()); - outputs.push_back(Y); - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/logsoftmax.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/logsoftmax.cc deleted file mode 100644 index aee2c5f78f..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/logsoftmax.cc +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/x86/x86_target_info.h" -#include "core/providers/nuphar/mti_x86/math/logsoftmax.h" - -namespace onnxruntime { -namespace nuphar { - -// Evaluate of LogSoftmax OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(LogSoftmax)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - - int64_t axis_i64; - ORT_RETURN_IF_ERROR(info.GetAttr("axis", &axis_i64)); - axis_i64 = HandleNegativeAxis(axis_i64, gsl::narrow_cast(inputs[0]->shape.size())); - - CodeGenTargetX86* target = dynamic_cast(ctx_codegen.GetCodeGenHandle()->codegen_target); - ORT_ENFORCE(target != nullptr); - int64_t natural_vector_size = target->NaturalVectorWidth(inputs[0]->dtype.bits()); - - tvm::Tensor Y = nuphar::LogSoftmax(inputs[0], axis_i64, natural_vector_size); - outputs.push_back(Y); - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/matmul.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/matmul.cc deleted file mode 100644 index 19ee25dce7..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/matmul.cc +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/codegen/mti/mti_tvm_utils.h" -#include "core/codegen/passes/weight_layout/transpose_2d.h" -#include "core/codegen/passes/weight_layout/vertical_stripes_2d.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" -#include "core/providers/nuphar/compiler/x86/x86_target_info.h" -#include "core/providers/nuphar/mti_x86/math/matmul_ops.h" - -#include - -namespace onnxruntime { -namespace nuphar { - -// TODO: remove tvm core function - -// local helper functions - -static bool MatMul_weights2D( - ONNX_NAMESPACE::TensorProto_DataType proto_type, - const tvm::Tensor& A, - const tvm::Tensor& B, - const std::string& initializer_name, - NupharCodeGenCtx& ctx_codegen, - tvm::Tensor& Y, - const std::string& name = "matmul_weights2d") { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - // optimizations for B being 2D weights - - // The 2D weight is marshalled with stripe_width. - // This should be 2x nature vector width - int stripe_width = 8; - int block_size = 32; - - onnxruntime::CodeGenTargetX86* target = - dynamic_cast(ctx_codegen.GetCodeGenHandle()->codegen_target); - if (nullptr != target) { - stripe_width = 2 * target->NaturalVectorWidth(B->dtype.bits()); - } - - // align A, B to multiple of block size - const auto& A_shape = A->shape; - tvm::Expr A0_size = tvm_codegen::SizeToDimension(A_shape, -1); - auto A0_roundup = tvm_codegen::RoundUp(A0_size, block_size); - tvm::Expr A1_size = tvm_codegen::SizeFromDimension(A_shape, -1); - auto A1_roundup = tvm_codegen::RoundUp(A1_size, block_size); - bool A0_need_pad = !tvm::ir::Equal(A0_roundup, A0_size); - bool A1_need_pad = !tvm::ir::Equal(A1_roundup, A1_size); - - const auto& B_shape = B->shape; - tvm::Expr B0_size = tvm_codegen::SizeToDimension(B_shape, 1); - auto B0_roundup = tvm_codegen::RoundUp(B0_size, block_size); - tvm::Expr B1_size = tvm_codegen::SizeFromDimension(B_shape, 1); - auto B1_roundup = tvm_codegen::RoundUp(B1_size, block_size); - bool B1_need_pad = !tvm::ir::Equal(B1_roundup, B1_size); - - ORT_ENFORCE(tvm::ir::Equal(A1_roundup, B0_roundup)); - - // Currently only support padding in B1, as it's free with memory marshalling - if (A0_need_pad || A1_need_pad || B1_need_pad) - return false; - - auto layout_key = tvm_codegen::WeightLayoutVerticalStripe2D::GetKey(proto_type, stripe_width); - auto B_unmarshalled = ctx_nuphar->ApplyWeightLayout(layout_key, initializer_name, B, false); - - ORT_ENFORCE(B_unmarshalled->op.as()); - - tvm::Array Y_shape; - for (size_t d = 0; d < A->shape.size() - 1; ++d) - Y_shape.push_back(A->shape[d]); - Y_shape.push_back(B->shape[1]); - - auto k = tvm::reduce_axis(tvm::Range(0, A1_size), "k"); - Y = tvm::compute( - Y_shape, - [&](const tvm::Array& idx) { - tvm::Array A_indices; - for (size_t d = 0; d < idx.size() - 1; ++d) - A_indices.push_back(idx[d]); - A_indices.push_back(k); - return tvm::sum(A(A_indices) * B_unmarshalled(k, idx[idx.size() - 1]), {k}); - }, - name); - - return true; -} - -static bool MatMulF32ExternCPU( - tvm::Tensor A, - tvm::Tensor B, - tvm::Tensor& Y, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - // try to fuse tranpose in MatMul input with MatMul - auto find_transposed_input = [&ctx_nuphar](const tvm::Tensor& t, std::vector& cumulated_permute) { - tvm::Tensor out = t; - int64_t rank = gsl::narrow(t->shape.size()); - std::vector default_node_perm(rank); - cumulated_permute.resize(rank); - for (int64_t i = 0; i < rank; ++i) { - cumulated_permute[i] = gsl::narrow(i); - default_node_perm[i] = rank - i - 1; - } - for (const Node* root_node = ctx_nuphar->FindNode(out); - root_node != nullptr && root_node->OpType() == "Transpose"; - root_node = ctx_nuphar->FindNode(out)) { - ProtoHelperNodeContext ctx(*root_node); - OpNodeProtoHelper info(&ctx); - auto perm = info.GetAttrsOrDefault("perm", default_node_perm); - std::vector updated_cumulated_permute = cumulated_permute; - for (int64_t dst_dim = 0; dst_dim < rank; ++dst_dim) { - auto src_dim = tvm_codegen::HandleNegativeAxis(perm[cumulated_permute[dst_dim]], rank); - updated_cumulated_permute[dst_dim] = gsl::narrow(src_dim); - } - cumulated_permute = updated_cumulated_permute; - // op corresponding to node should be Transpose - auto op = out->op.as(); - ORT_ENFORCE(op != nullptr); - ORT_ENFORCE(op->InputTensors().size() == 1); - out = op->InputTensors()[0]; - } - return out; - }; - - std::vector permute_A; - std::vector permute_B; - const std::vector* p_permute_A = nullptr; - const std::vector* p_permute_B = nullptr; - tvm::Tensor root_A = find_transposed_input(A, permute_A); - tvm::Tensor root_B = find_transposed_input(B, permute_B); - bool transA = false; - if (A->shape.size() == B->shape.size() && A->shape.size() >= 2) { - // currently only fuse Transpose into MatMul when rank(A) == rank(B) - // make sure no broadcasting in MatMul - bool no_broadcast = true; - for (size_t i = 0; i < A->shape.size() - 2; ++i) { - if (!tvm::ir::Equal(A->shape[i], B->shape[i])) { - no_broadcast = false; - break; - } - } - if (no_broadcast) { - if (CanPermuteBeFusedInMatMul(permute_A)) { - if (A != root_A) - transA = true; - A = root_A; - p_permute_A = &permute_A; - } - if (CanPermuteBeFusedInMatMul(permute_B)) { - B = root_B; - p_permute_B = &permute_B; - } - } - } - - const auto& B_name = node.InputDefs()[1]->Name(); - if (ctx_nuphar->IsInitializer(B_name) && B->shape.size() == 2) { - if (A->shape.size() == 1) { - return nuphar::GemmExternCpu(A, B, Y, transA, false, B_name); - } else { - // matmul with initializer, using transpose weights - auto layout_key = tvm_codegen::WeightLayoutTranspose2D::GetKey(ONNX_NAMESPACE::TensorProto_DataType_FLOAT); - auto actual_B = ctx_nuphar->ApplyWeightLayout(layout_key, B_name, B, true); - return nuphar::GemmExternCpu(A, actual_B, Y, transA, true, B_name); - } - } else { - return nuphar::MatMulExternCpu(A, B, Y, p_permute_A, p_permute_B, node.Name() + "_matmul_extern"); - } -} - -Status -NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(MatMul)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - auto proto_type = TensorProtoDataType(node.InputDefs()[1]); - - tvm::Tensor Y; - auto& A = inputs[0]; - auto& B = inputs[1]; - - // float MatMul, try use extern - if (A->dtype == HalideIR::Float(32) && - B->dtype == HalideIR::Float(32) && - MatMulF32ExternCPU(A, B, Y, node, ctx_codegen)) { - outputs.push_back(Y); - return Status::OK(); - } - - // if B is 2D initializer, use vertical stripe layout - const std::string& input_1_name = node.InputDefs()[1]->Name(); - if (ShapeRank(node.InputDefs()[1]) == 2 && ctx_nuphar->IsInitializer(input_1_name)) { - if (MatMul_weights2D(proto_type, A, B, input_1_name, *ctx_nuphar, Y)) { - outputs.push_back(Y); - return Status::OK(); - } - } - - Y = nuphar::MatMul(A, B, node.Name()); - outputs.push_back(Y); - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/pow.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/pow.cc deleted file mode 100644 index 7b1d04d1a9..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/pow.cc +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/framework/op_kernel_info.h" -#include "core/providers/nuphar/common/nuphar_tvm_utils.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" -#include "core/providers/nuphar/mti_x86/math/pow.h" - -namespace onnxruntime { -namespace nuphar { - -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Pow)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - ORT_ENFORCE(inputs.size() == 2); - - struct { - tvm::Expr expr; - bool is_scalar; - } constant_scalars[2]; - - for (size_t i = 0; i < 2; ++i) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - ORT_ENFORCE(i < node.InputDefs().size()); - const auto* tensor = ctx_nuphar->GetOrtInitializerTensor(node.InputDefs()[i]->Name()); - constant_scalars[i].is_scalar = TryCreateConstantScalar(constant_scalars[i].expr, tensor); - } - tvm::Tensor Y; - if (constant_scalars[0].is_scalar) - Y = Pow(constant_scalars[0].expr, inputs[1], node.Name()); - else if (constant_scalars[1].is_scalar) - Y = Pow(inputs[0], constant_scalars[1].expr, node.Name()); - else - Y = Pow(inputs[0], inputs[1], node.Name()); - outputs.push_back(Y); - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/quantize/matmul_integer.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/quantize/matmul_integer.cc deleted file mode 100644 index a4f5c75ae6..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/quantize/matmul_integer.cc +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/codegen/common/utils.h" -#include "core/codegen/mti/math/binary_ops.h" -#include "core/codegen/mti/math/matmul_ops.h" -#include "core/codegen/mti/mti_tvm_utils.h" -#include "core/codegen/mti/tensor/cast_ops.h" -#include "core/codegen/mti/tensor/pad_ops.h" -#include "core/codegen/mti/tensor/reshape_ops.h" -#include "core/codegen/mti/tensor/transpose.h" -#include "core/codegen/passes/weight_layout/tiling_2d.h" -#include "core/codegen/passes/weight_layout/transpose_2d.h" -#include "core/common/cpuid_info.h" // TODO: refactor to control through config -#include "core/providers/nuphar/common/nuphar_settings.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/compiler/x86/x86_target_info.h" -#include "core/providers/nuphar/mti_x86/quantize/imatmul_extern.h" -#include "core/providers/nuphar/mti_x86/quantize/imatmul16_extern.h" - -namespace onnxruntime { -namespace nuphar { - -tvm::Tensor IMatMulTensorize(const tvm::Tensor& A, - const tvm::Tensor& B, - const tvm::Expr& batchseq_dim, - int input_dim, - int embed_dim, - int vector_width, - const std::string& name) { - tvm::Tensor A_reshape; - int A_rank = gsl::narrow_cast(A->shape.size()); - ORT_ENFORCE(A_rank >= 2); - - const int64_t* A_dim0 = tvm::as_const_int(A->shape[0]); - const int64_t* A_dim1 = tvm::as_const_int(A->shape[1]); - const int64_t* B_dim0 = tvm::as_const_int(B->shape[0]); - const int64_t* B_dim1 = tvm::as_const_int(B->shape[1]); - ORT_ENFORCE(B_dim1 != nullptr && B_dim0 != nullptr); - - int input_padded = gsl::narrow_cast(*B_dim1); - int embed_padded = gsl::narrow_cast(*B_dim0); - const int64_t* p_batchseq_dim = tvm::as_const_int(batchseq_dim); - - if (p_batchseq_dim != nullptr && A_rank == 2 && - A_dim0 != nullptr && *A_dim0 == *p_batchseq_dim && - A_dim1 != nullptr && gsl::narrow_cast(*A_dim1) == input_dim) { - A_reshape = A; - } else { - A_reshape = tvm_codegen::Reshape(A, {batchseq_dim, input_dim}, name + "_reshape_X"); - } - - if (input_dim != input_padded) { - tvm::Expr pad_value = tvm::make_const(A->dtype, 0); - A_reshape = tvm_codegen::PadLastDim(A_reshape, vector_width, pad_value); - } - - tvm::Tensor Y = tvm::compute( - {batchseq_dim, embed_padded}, - [&](const tvm::Array& indices) { - auto k = tvm::reduce_axis({0, input_padded}); - return tvm::sum(tvm::cast(HalideIR::Int(32), A_reshape(indices[0], k)) * tvm::cast(HalideIR::Int(32), B(indices[1], k)), {k}); - }, - name + "_tensorize_Y"); - return Y; -} - -// A generic path, cast to int32 -// Support skipped trailing inputs -tvm::Tensor GenericMatMulInteger(const tvm::Array& inputs, const Node& node) { - auto A_Int32 = (node.InputDefs().size() >= 3 && node.InputDefs()[2]->Exists()) - ? tvm_codegen::Sub(tvm_codegen::Cast(inputs[0], HalideIR::Int(32)), tvm_codegen::Cast(inputs[2], HalideIR::Int(32))) - : tvm_codegen::Cast(inputs[0], HalideIR::Int(32)); - auto B_Int32 = (node.InputDefs().size() >= 4 && node.InputDefs()[3]->Exists()) - ? tvm_codegen::Sub(tvm_codegen::Cast(inputs[1], HalideIR::Int(32)), tvm_codegen::Cast(inputs[3], HalideIR::Int(32))) - : tvm_codegen::Cast(inputs[1], HalideIR::Int(32)); - return tvm_codegen::MatMul(A_Int32, B_Int32, node.Name() + "_Generic"); -} - -// Evaluate of MatMulInteger -static Status EvaluateMatMulInteger( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - const auto& A = inputs[0]; - const auto& B = inputs[1]; - auto& name = node.Name(); - - bool is8bit = (A->dtype == HalideIR::type_of() && - B->dtype == HalideIR::type_of()); - - if (B->shape.size() == 2 && is8bit) { - const int64_t* p_input_dim = tvm::as_const_int(B->shape[0]); - const int64_t* p_embed_dim = tvm::as_const_int(B->shape[1]); - - if (p_input_dim != nullptr && p_embed_dim != nullptr) { - int64_t input_dim = *p_input_dim; - int64_t embed_dim = *p_embed_dim; - - // batchseq_dim: batch * seq - auto A_rank = gsl::narrow_cast(A->shape.size()); - auto batchseq_dim = tvm_codegen::SizeToDimension(A->shape, A_rank - 1); - const int64_t* p_batch_seq_dim = tvm::as_const_int(batchseq_dim); - - tvm::Array output_shape; - for (int i = 0; i < A_rank - 1; ++i) { - output_shape.push_back(A->shape[i]); - } - output_shape.push_back(tvm::Expr(gsl::narrow_cast(embed_dim))); - - // Enviornment variables option - const codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - TargetFeature feature = GetTargetInfo(settings); - - bool force_mkl = false; - if (settings.HasOption(kNupharIMatMulForceMkl)) { - force_mkl = true; - } - bool force_no_tensorize = false; - if (settings.HasOption(kNupharForceNoTensorize)) { - force_no_tensorize = true; - } - - // Tensorization: AVX2: 8bit GEMV/GEMM AVX512: 8bit GEMV/GEMM - bool use_tensorization = !force_mkl && !force_no_tensorize && (feature.hasAVX512 || feature.hasAVX2 || feature.hasAVX); - - // Model input option - auto B_NodeArg = node.InputDefs()[1]; - const std::string& B_name = B_NodeArg->Name(); - bool hasInitializer = ctx_nuphar->IsInitializer(B_name); - - if (!hasInitializer) { - //TODO: change to use MLAS when no layout could apply - tvm::Tensor B_marshalled = tvm_codegen::Transpose(B, {1, 0}); - - bool use_extern_MKL = (force_mkl || !feature.hasAVX2); - tvm::Tensor output_tensor = use_extern_MKL ? IMatMulExternMKL(A, B_marshalled, output_shape, input_dim, embed_dim, name + "_IMatMulExternMKL") - : IMatMulExternAVX2(A, B_marshalled, output_shape, input_dim, embed_dim, name + "_IMatMulExternAVX2"); - - outputs.push_back(output_tensor); - } else if (use_tensorization) { - // vector width determined from target hardware - // AVX: vector width 16 = 128 bits / 8 bits; - // AVX2: vector width 32 = 256 bits / 8 bits; - // AVX512: vector width 64 = 512 bits / 8 bits; - CodeGenTargetX86* target = dynamic_cast(ctx_codegen.GetCodeGenHandle()->codegen_target); - ORT_ENFORCE(target != nullptr, "CodeGen target unknown: not AVX/AVX2/AVX512 !"); - int vector_width = target->NaturalVectorWidth(B->dtype.bits()) / 2; - - // TVM has known issue when handling tensorization of matmul: [1x1] = [1xK]x[Kx1] - // and this case is not likely happen in real model - // so add option to fall back to a general reduction - bool isScalar = (p_batch_seq_dim != nullptr && *p_batch_seq_dim == 1) && (embed_dim == 1); - - // Tensorization has two layout options: 1) Transpose or 2) Tiling - auto layout_key = isScalar ? tvm_codegen::WeightLayoutTranspose2D::GetKey(TensorProtoDataType(B_NodeArg)) : tvm_codegen::WeightLayoutTiling2D::GetKey(TensorProtoDataType(B_NodeArg), vector_width); - - tvm::Tensor B_marshalled = ctx_nuphar->ApplyWeightLayout(layout_key, B_name, B, true); - - tvm::Tensor output_tensor = IMatMulTensorize(A, B_marshalled, batchseq_dim, input_dim, embed_dim, vector_width, name + "_IMatMulTensorize"); - - // Post processing output tensor - tvm::Expr embed_padded = B_marshalled->shape[0]; - const int64_t* p_embed_padded = tvm::as_const_int(embed_padded); - ORT_ENFORCE(p_embed_padded != nullptr); - - tvm::Array Y_shape; - for (int i = 0; i < A_rank - 1; ++i) { - Y_shape.push_back(A->shape[i]); - } - Y_shape.push_back(embed_padded); - - tvm::Tensor Y = tvm_codegen::Reshape(output_tensor, Y_shape, name + "_reshape"); - - if (*p_embed_padded != embed_dim) { - tvm::Tensor Y_unpad = tvm::compute(output_shape, - [&](const tvm::Array& indices) { - return Y(indices); - }, - name + "_unpad_shape"); - outputs.push_back(Y_unpad); - } else { - outputs.push_back(Y); - } - } else { - auto layout_key = tvm_codegen::WeightLayoutTranspose2D::GetKey(TensorProtoDataType(B_NodeArg)); - tvm::Tensor B_marshalled = ctx_nuphar->ApplyWeightLayout(layout_key, B_name, B, true); - - bool use_extern_AVX2 = (!force_mkl && feature.hasAVX2); - tvm::Tensor output_tensor = use_extern_AVX2 ? IMatMulExternAVX2(A, B_marshalled, output_shape, input_dim, embed_dim, name + "_IMatMulExternAVX2") - : IMatMulExternMKL(A, B_marshalled, output_shape, input_dim, embed_dim, name + "_IMatMulExternMKL"); - - outputs.push_back(output_tensor); - } - - return Status::OK(); - } - } - - tvm::Tensor Y = GenericMatMulInteger(inputs, node); - outputs.push_back(Y); - return Status::OK(); -} - -// Evaluate of MatMulInteger16 -static Status EvaluateMatMulInteger16( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - const auto& A = inputs[0]; - const auto& B = inputs[1]; - - bool is16bit = (A->dtype == HalideIR::type_of() && - B->dtype == HalideIR::type_of()); - - const codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - TargetFeature feature = GetTargetInfo(settings); - - // 16bit on AVX fall back to 32bit - bool AVXonly = feature.hasAVX && !feature.hasAVX2; - - if (!AVXonly && (B->shape.size() == 2 && is16bit)) { - const int64_t* p_input_dim = tvm::as_const_int(B->shape[0]); - const int64_t* p_embed_dim = tvm::as_const_int(B->shape[1]); - - if (p_input_dim != nullptr && p_embed_dim != nullptr) { - int64_t input_dim = *p_input_dim; - int64_t embed_dim = *p_embed_dim; - - auto A_rank = gsl::narrow_cast(A->shape.size()); - tvm::Array output_shape; - for (int i = 0; i < A_rank - 1; ++i) { - output_shape.push_back(A->shape[i]); - } - output_shape.push_back(tvm::Expr(gsl::narrow_cast(embed_dim))); - - bool force_mkl = false; - if (settings.HasOption(kNupharIMatMulForceMkl)) { - force_mkl = true; - } - - // Model input option - auto B_NodeArg = node.InputDefs()[1]; - const std::string& B_name = B_NodeArg->Name(); - bool hasInitializer = ctx_nuphar->IsInitializer(B_name); - - if (!hasInitializer) { - //TODO: change to use MLAS when no layout could apply - tvm::Tensor B_marshalled = tvm_codegen::Transpose(B, {1, 0}); - - bool use_extern_MKL = (force_mkl || !feature.hasAVX2); - tvm::Tensor output_tensor = use_extern_MKL ? IMatMul16ExternMKL(A, B_marshalled, output_shape, input_dim, embed_dim, node.Name() + "_IMatMulExternMKL") - : IMatMul16ExternAVX2(A, B_marshalled, output_shape, input_dim, embed_dim, node.Name() + "_IMatMulExternAVX2"); - outputs.push_back(output_tensor); - } else { - auto layout_key = tvm_codegen::WeightLayoutTranspose2D::GetKey(TensorProtoDataType(B_NodeArg)); - tvm::Tensor B_marshalled = ctx_nuphar->ApplyWeightLayout(layout_key, B_name, B, true); - - bool use_extern_AVX2 = (!force_mkl && feature.hasAVX2); - tvm::Tensor output_tensor = use_extern_AVX2 ? IMatMul16ExternAVX2(A, B_marshalled, output_shape, input_dim, embed_dim, node.Name() + "_IMatMulExternAVX2") - : IMatMul16ExternMKL(A, B_marshalled, output_shape, input_dim, embed_dim, node.Name() + "_IMatMulExternMKL"); - outputs.push_back(output_tensor); - } - - return Status::OK(); - } - } - - tvm::Tensor Y = GenericMatMulInteger(inputs, node); - outputs.push_back(Y); - return Status::OK(); -} - -// Evaluate of MatMulInteger OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(MatMulInteger)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - return EvaluateMatMulInteger(inputs, node, ctx_codegen, outputs); -} - -// Evaluate of MatMulInteger16 OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(MatMulInteger16)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - return EvaluateMatMulInteger16(inputs, node, ctx_codegen, outputs); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/reduce_ops.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/reduce_ops.cc deleted file mode 100644 index 55d3a08678..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/reduce_ops.cc +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/mti_x86/math/reduce_ops.h" - -#include // for sort - -namespace onnxruntime { -namespace nuphar { - -using ReduceVFunc = tvm::Tensor (*)(const tvm::Tensor& X, - const std::vector& axes, - bool keep_dims, - int32_t vector_size, - bool last_dim_aligned, - int32_t fuse_dim, - const std::string& name); - -// This function gives a proper vector width and fuse dim for reduce -// It avoids vector_width larger than shape -// Fuse dim implies mulitple reduce axis could be fused together to form a longer vector_width -// It can avoid too small vector_width -static std::tuple VectorWidthAndFuseDimForReduce(int natural_width, - std::vector axes, - const NodeArg* def) { - int64_t rank = ShapeRank(def); - if (rank == 0) { - return std::make_tuple(1, 0); - } - - int tail_size = 1; - - // reduce all - if (axes.size() == 0) { - for (int i = gsl::narrow_cast(rank) - 1; i >= 0; --i) { - if (ShapeHasValue(def, i)) { - tail_size *= gsl::narrow_cast(ShapeValue(def, i)); - } else { - if (i > 0) - return std::make_tuple(tail_size, i - 1); - else - return std::make_tuple(natural_width, 0); - } - - if (tail_size >= natural_width) { - return std::make_tuple(natural_width, i); - } - } - - return std::make_tuple(tail_size, 0); - } - - //reduce last - int j = axes.size() - 1; - if (axes.back() == (rank - 1)) { - for (int i = gsl::narrow_cast(rank) - 1; i >= 0; --i) { - if (ShapeHasValue(def, i) && axes[j] == gsl::narrow_cast(i)) { - tail_size *= gsl::narrow_cast(ShapeValue(def, i)); - if (j > 0) - --j; - } else { - if (i > 0) { - return std::make_tuple(tail_size, i - 1); - } else { - return std::make_tuple(natural_width, 0); - } - } - - if (tail_size >= natural_width) { - return std::make_tuple(natural_width, i); - } - } - - return std::make_tuple(tail_size, 0); - } - - // reduce other - for (int i = gsl::narrow_cast(rank) - 1; i >= 0; --i) { - if (ShapeHasValue(def, i) && axes[j] != gsl::narrow_cast(i)) { - tail_size *= gsl::narrow_cast(ShapeValue(def, i)); - if (j > 0) - --j; - } else { - if (i > 0) - return std::make_tuple(tail_size, i - 1); - else - return std::make_tuple(natural_width, 0); - } - - if (tail_size >= natural_width) { - return std::make_tuple(natural_width, i); - } - } - - return std::make_tuple(tail_size, 0); -} - -class FuncReduceV { - public: - FuncReduceV(NupharCodeGenCtx* ctx_nuphar, - const Node& node, - ReduceVFunc func, - std::function natural_vector, - const NodeArg* def, - const std::string& name) : def_(def) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - int version = ctx_nuphar->GetCodeGenHandle()->domain_version_lookup_func(node.Domain()); - if (node.OpType() == "ReduceSum" && version >= 13) { - // ReduceSum changed axes from attribute to input in opset 13 - // besides, it added noop_with_empty_axes attribute to do nothing when axes is empty - const auto& inputs = node.InputDefs(); - if (inputs.size() > 1 && inputs[1] != nullptr) { - const auto* tensor = ctx_nuphar->GetOrtInitializerTensor(inputs[1]->Name()); - ORT_ENFORCE(tensor); - const int64_t* input_axes = tensor->Data(); - axes_ = std::vector(input_axes, input_axes + tensor->Shape().Size()); - } - noop_with_empty_axes_ = (info.GetAttrOrDefault("noop_with_empty_axes", 0) != 0); - } else { - axes_ = info.GetAttrsOrDefault("axes"); - noop_with_empty_axes_ = false; - } - - if (!noop_with_empty_axes_ && axes_.size() == 0) { - int64_t sz = static_cast(def->Shape()->dim().size()); - ORT_ENFORCE(sz > 0); - for (int64_t i = 0; i < sz; i++) { - axes_.push_back(i); - } - } - - int64_t keepdims_i = 1; - ORT_ENFORCE(info.GetAttr("keepdims", &keepdims_i).IsOK()); - keep_dims_ = (keepdims_i == 1); - func_ = func; - name_ = node.Name() + "_" + name; - natural_vector_ = natural_vector; - } - - tvm::Tensor operator()(const tvm::Tensor& X) const { - if (noop_with_empty_axes_ && axes_.size() == 0) { - return X; // No-op when noop_with_empty_axes is true and axes is empty (ReduceSum-13) - } - - std::vector axes; - for (auto i : axes_) { - axes.push_back(HandleNegativeAxis(i, gsl::narrow_cast(X->shape.size()))); - } - - std::sort(axes.begin(), axes.end()); //ReduceV requires sorted axes - - auto p = VectorWidthAndFuseDimForReduce(natural_vector_(X->dtype.bits()), axes, def_); - int vector_width = std::get<0>(p); - int fuse_dim = std::get<1>(p); - - bool last_dim_aligned = false; - const int64_t* p_last_dim_size = tvm::as_const_int(X->shape[X->shape.size() - 1]); - - if (p_last_dim_size != nullptr) { - last_dim_aligned = (*p_last_dim_size) % vector_width == 0; - } - - return func_(X, axes, keep_dims_, vector_width, last_dim_aligned, fuse_dim, name_); - } - - private: - std::vector axes_; - bool keep_dims_; - ReduceVFunc func_; - std::string name_; - std::function natural_vector_; - const NodeArg* def_; - bool noop_with_empty_axes_; -}; - -#define REDUCE_V_OP(name) \ - Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(name)::Evaluate( \ - const tvm::Array& inputs, \ - const Node& node, \ - tvm_codegen::CodeGenContext& ctx_codegen, \ - tvm::Array& outputs) { \ - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); \ - auto natural_vector = [&](int bits) { \ - return ctx_codegen.GetCodeGenHandle()->codegen_target->NaturalVectorWidth(bits); \ - }; \ - tvm::Tensor Y = FuncReduceV(ctx_nuphar, node, &nuphar::name, natural_vector, node.InputDefs()[0], #name)(inputs[0]); \ - outputs.push_back(Y); \ - return Status::OK(); \ - } - -LIST_REDUCE_V_OPS() - -#undef REDUCE_V_OP - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/softmax.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/softmax.cc deleted file mode 100644 index a3e1ea5077..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/softmax.cc +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/x86/x86_target_info.h" -#include "core/providers/nuphar/mti_x86/math/softmax.h" - -namespace onnxruntime { -namespace nuphar { - -// Evaluate of Softmax OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Softmax)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - - int64_t axis_i64; - ORT_RETURN_IF_ERROR(info.GetAttr("axis", &axis_i64)); - axis_i64 = HandleNegativeAxis(axis_i64, gsl::narrow_cast(inputs[0]->shape.size())); - - CodeGenTargetX86* target = dynamic_cast(ctx_codegen.GetCodeGenHandle()->codegen_target); - ORT_ENFORCE(target != nullptr); - int64_t natural_vector_size = target->NaturalVectorWidth(inputs[0]->dtype.bits()); - - tvm::Tensor Y = Softmax(inputs[0], axis_i64, natural_vector_size); - outputs.push_back(Y); - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/unary_ops.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/unary_ops.cc deleted file mode 100644 index 30fec1dc24..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/math/unary_ops.cc +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/codegen/passes/op_ir_creator/math/unary_funcs.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/nuphar/mti_x86/math/unary_ops.h" - -namespace onnxruntime { -namespace nuphar { - -// helper macro declares unary_ops helper class without attribute -#define FuncClass(name) \ - class Func##name { \ - public: \ - Func##name(const Node&) {} \ - tvm::Tensor operator()(const tvm::Tensor& X) const { \ - return name(X); \ - } \ - } - -// helper macro declares unary_ops helper class with alpha -#define FuncClassAlpha(name) \ - class Func##name : public tvm_codegen::FuncWithAlpha { \ - public: \ - Func##name(const Node& node) : FuncWithAlpha(node) {} \ - tvm::Tensor operator()(const tvm::Tensor& X) const { \ - return name(X, alpha_); \ - } \ - } - -// helper macro declares unary_ops helper class with alpha and beta -#define FuncClassAlphaBeta(name) \ - class Func##name : public tvm_codegen::FuncWithAlphaBeta { \ - public: \ - Func##name(const Node& node) : FuncWithAlphaBeta(node) {} \ - tvm::Tensor operator()(const tvm::Tensor& X) const { \ - return name(X, alpha_, beta_); \ - } \ - } - -// helper macro declares unary_ops helper class with alpha and gamma -#define FuncClassAlphaGamma(name) \ - class Func##name : public tvm_codegen::FuncWithAlphaGamma { \ - public: \ - Func##name(const Node& node) : FuncWithAlphaGamma(node) {} \ - tvm::Tensor operator()(const tvm::Tensor& X) const { \ - return name(X, alpha_, gamma_); \ - } \ - } - -FuncClass(Erf); -FuncClass(Exp); -FuncClass(Log); -FuncClassAlphaBeta(ParametricSoftplus); -FuncClassAlphaBeta(ScaledTanh); -FuncClassAlphaGamma(Selu); -FuncClass(Sigmoid); -FuncClass(Softplus); -FuncClass(Tanh); - -// helper macro defines Evaluate of UNARY_OP OpIRCreators -#define UNARY_OP(name) \ - Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(name)::Evaluate( \ - const tvm::Array& inputs, \ - const Node& node, \ - tvm_codegen::CodeGenContext&, \ - tvm::Array& outputs) { \ - tvm::Tensor Y = Func##name(node)(inputs[0]); \ - outputs.push_back(Y); \ - return Status::OK(); \ - } - -// helper local macros to replace some calls in LIST_UNARY_OPS -LIST_X86_UNARY_OPS() - -#undef UNARY_OP - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/nn/pool_ops.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/nn/pool_ops.cc deleted file mode 100644 index 47aa059e59..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/nn/pool_ops.cc +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/codegen/passes/utils/ort_tvm_utils.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/cpu/nn/pool_attributes.h" -#include "core/providers/nuphar/mti_x86/nn/pool_ops.h" - -namespace onnxruntime { -namespace nuphar { - -static tvm::Array GetOutputShapeAndPads(const Node& node, - PoolAttributes& pool_attrs, - tvm_codegen::CodeGenContext& ctx_codegen) { - const NodeArg* input = node.InputDefs()[0]; - ORT_ENFORCE(input); - const ONNX_NAMESPACE::TensorShapeProto* shape_proto = input->Shape(); - size_t num_input_dims = shape_proto->dim_size(); - ORT_ENFORCE(num_input_dims >= 2); - - tvm::Array output_shape; - // batch dimenion - output_shape.push_back(ShapeDimToTvmDim(shape_proto->dim(0), ctx_codegen)); - // output channel - output_shape.push_back(ShapeDimToTvmDim(shape_proto->dim(1), ctx_codegen)); - - size_t kernel_sz = pool_attrs.kernel_shape.size(); - if (pool_attrs.global_pooling) { - pool_attrs.pads.assign(kernel_sz, 0); - // skip batch and channel dimensions, so dim starts from 2 - for (size_t dim = 2; dim < num_input_dims; dim++) { - output_shape.push_back(tvm::make_const(tvm::Int(32), 1)); - } - } else { - ORT_ENFORCE(num_input_dims > kernel_sz); - size_t kernel_idx_offset = num_input_dims - kernel_sz; - for (size_t dim = 0; dim < kernel_sz; dim++) { - // TODO: handle symbolic dimensions - ORT_ENFORCE(ShapeHasValue(input, dim + kernel_idx_offset)); - int64_t dim_val = ShapeValue(input, dim + kernel_idx_offset); - int64_t dim_size = 0; - pool_attrs.ComputeSizePadDilations(static_cast(dim_val), - pool_attrs.strides[dim], - pool_attrs.kernel_shape[dim], - &(pool_attrs.pads[dim]), - &(pool_attrs.pads[kernel_sz + dim]), - pool_attrs.dilations[dim], - &dim_size); - output_shape.push_back(tvm::make_const(tvm::Int(32), dim_size)); - } - } - return output_shape; -} - -#define POOL_OP(name) \ - Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(name)::Evaluate( \ - const tvm::Array& inputs, \ - const Node& node, \ - tvm_codegen::CodeGenContext& ctx_codegen, \ - tvm::Array& outputs) { \ - ORT_RETURN_IF_NOT(node.OutputDefs().size() == 1, " multiple outputs are not supported yet!"); \ - ORT_RETURN_IF_NOT(inputs[0]->dtype == HalideIR::Float(32), " non-float32 not supported yet"); \ - ProtoHelperNodeContext ctx(node); \ - OpNodeProtoHelper info(&ctx); \ - int version = ctx_codegen.GetCodeGenHandle()->domain_version_lookup_func(node.Domain()); \ - PoolAttributes pool_attrs(info, #name, version); \ - for (auto n : pool_attrs.dilations) { \ - ORT_RETURN_IF_NOT(n <= 1, "dilations are not supported yet!"); \ - } \ - tvm::Array output_shape = GetOutputShapeAndPads(node, pool_attrs, ctx_codegen); \ - tvm::Tensor Y = name(inputs[0], pool_attrs, output_shape); \ - outputs.push_back(Y); \ - return Status::OK(); \ - } \ - -LIST_X86_POOL_OPS() - -#undef POOL_OP - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/dropout.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/dropout.cc deleted file mode 100644 index 6d1cdcf665..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/dropout.cc +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/codegen/mti/mti_tvm_utils.h" -#include "core/codegen/mti/tensor/reshape_ops.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" - -namespace onnxruntime { -namespace nuphar { - -// Evaluate of Dropout OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Dropout)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - tvm::Tensor Y = tvm_codegen::Identity(inputs[0]); - outputs.push_back(Y); - - // optional mask - // Support skipped trailing outputs - if (node.OutputDefs().size() > 1 && node.OutputDefs()[1]->Exists()) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - int version = ctx_nuphar->GetCodeGenHandle()->domain_version_lookup_func(node.Domain()); - if (version >= 12) { - // A fake mask with all ones for opset 12+ - auto l = [&](const tvm::Array& /*indices*/) { - return tvm::make_const(tvm::UInt(8), 1); - }; - tvm::Tensor mask = tvm::compute(inputs[0]->shape, l, "mask"); - outputs.push_back(mask); - } else { - // for opset < 12, masks are all zero - tvm::Tensor mask = tvm_codegen::MakeZeroTensor(inputs[0]->shape, inputs[0]->dtype, "mask"); - outputs.push_back(mask); - } - } - - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/scatter.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/scatter.cc deleted file mode 100644 index c33afb3bbc..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/scatter.cc +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/framework/op_kernel_info.h" -#include "core/providers/nuphar/mti_x86/tensor/scatter.h" - -namespace onnxruntime { -namespace nuphar { - -static Status ScatterCommon( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext&, - tvm::Array& outputs, - const std::string& name) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper attrs(&ctx); - - // The default value of optional attribute axis is 0 - int64_t axis = attrs.GetAttrOrDefault("axis", 0); - - tvm::Tensor Y = Scatter(inputs[0], axis, inputs[1], inputs[2], name); - outputs.push_back(Y); - return Status::OK(); -} - -// Evaluate of Scatter OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Scatter)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& codegen_ctx, - tvm::Array& outputs) { - return ScatterCommon(inputs, node, codegen_ctx, outputs, node.Name() + "_ScatterElements"); -} - -// Evaluate of ScatterElements OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(ScatterElements)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& codegen_ctx, - tvm::Array& outputs) { - return ScatterCommon(inputs, node, codegen_ctx, outputs, node.Name() + "_Scatter"); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/slice.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/slice.cc deleted file mode 100644 index 6d3a32eb7d..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/slice.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/codegen/mti/tensor/tile.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" - -namespace onnxruntime { -namespace tvm_codegen { - -// Forwarding -Status SliceCommon(const tvm::Array& inputs, - const Node& node, - tvm::Array& outputs, - const std::vector& starts, - const std::vector& ends, - const std::vector& axes, - const std::vector& steps); - -} // namespace tvm_codegen - -namespace nuphar { - -// Evaluate of Slice OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Slice)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - std::vector> slice_params; - int version = ctx_codegen.GetCodeGenHandle()->domain_version_lookup_func(node.Domain()); - if (version <= 9) { - std::vector starts, ends, axes, steps; - ORT_RETURN_IF_ERROR(info.GetAttrs("starts", starts)); - ORT_RETURN_IF_ERROR(info.GetAttrs("ends", ends)); - ORT_RETURN_IF_NOT(starts.size() == ends.size(), "starts.size() != ends.size()"); - axes = info.GetAttrsOrDefault("axes"); - slice_params.push_back(starts); - slice_params.push_back(ends); - slice_params.push_back(axes); - slice_params.push_back(steps); - } else { - // for opset 10 Slice, input 1/2/3/4 are starts/ends/axes/steps - // while axes and steps are optional - for (size_t i = 1; i < 5; ++i) { - if (i < node.InputDefs().size()) { - const auto* tensor = ctx_nuphar->GetOrtInitializerTensor(node.InputDefs()[i]->Name()); - if (tensor) { - if (tensor->IsDataType()) { - const int64_t* data = tensor->Data(); - slice_params.push_back(std::vector(data, data + tensor->Shape().Size())); - } else { - const int32_t* data = tensor->Data(); - slice_params.push_back(std::vector(data, data + tensor->Shape().Size())); - } - continue; - } - } - slice_params.push_back(std::vector()); - } - } - return tvm_codegen::SliceCommon(inputs, node, outputs, slice_params[0], slice_params[1], slice_params[2], slice_params[3]); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/tile.cc b/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/tile.cc deleted file mode 100644 index e96fb06f87..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/op_ir_creator/tensor/tile.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/op_ir_creator/all_ops.h" - -#include "core/codegen/mti/tensor/tile.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/common.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" - -namespace onnxruntime { -namespace nuphar { - -// Evaluate of Tile OpIRCreator -Status NUPHAR_TVM_X86_OP_IR_CREATOR_CLASS(Tile)::Evaluate( - const tvm::Array& inputs, - const Node& node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm::Array& outputs) { - ProtoHelperNodeContext ctx(node); - OpNodeProtoHelper info(&ctx); - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - const auto* repeats = ctx_nuphar->GetOrtInitializerTensor(node.InputDefs()[1]->Name()); - ORT_RETURN_IF_NOT(repeats != nullptr, "repeats == nullptr"); - ORT_RETURN_IF_NOT(repeats->Shape().Size() == gsl::narrow(inputs[0]->shape.size()), - "repeats->Shape().Size() != inputs[0]->shape.size()"); - const int64_t* repeats_data = repeats->Data(); - const auto repeats_vector = std::vector(repeats_data, repeats_data + inputs[0]->shape.size()); - tvm::Tensor Y = tvm_codegen::Tile(inputs[0], repeats_vector, node.Name() + "_Tile"); - outputs.push_back(Y); - return Status::OK(); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/analysis_schedule.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/analysis_schedule.cc deleted file mode 100644 index 14ce7e33aa..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/analysis_schedule.cc +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h" - -#include "core/codegen/passes/scheduler/schedule_utils.h" - -namespace onnxruntime { -namespace nuphar { - -// This is for UseCount -bool TVM_SCHEDULER_CLASS(True, NupharX86UseCount)::Evaluate( - const tvm::Tensor& tensor, - const Node*, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - bool status_vec = TryVectorizationX86(tensor, ctx_codegen, ctx_sched); - bool status_r_and_c = tvm_codegen::InsertRootScheduleAndClosure(tensor, ctx_sched); - return status_vec || status_r_and_c; -} - -bool TVM_SCHEDULER_CLASS(False, NupharX86UseCount)::Evaluate( - const tvm::Tensor& tensor, - const Node*, - tvm_codegen::CodeGenContext&, - tvm_codegen::ScheduleContext& ctx_sched) { - return tvm_codegen::TryInlineSchedule(tensor, ctx_sched); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.cc deleted file mode 100644 index 13285e7aa3..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.cc +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h" - -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" - -namespace onnxruntime { -namespace nuphar { - -tvm_codegen::Scheduler* SCHEDULE_DISPATCHER_CLASS(NupharX86UseCount):: - Find(const tvm::Tensor&, const Node* node, tvm_codegen::CodeGenContext& ctx) { - if (nullptr == node) - return nullptr; - - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx); - bool reused = Promote(ctx_nuphar->GetGraphStats())->NodeUseCount(node) > 1; - bool cheap_node_reused = Promote(ctx_nuphar->GetGraphStats())->IsCheapNodeReuse(node); - - if (reused && cheap_node_reused) { - return DispatcherBase::Get("True"); - } - return DispatcherBase::Get("False"); -} - -tvm_codegen::Scheduler* SCHEDULE_DISPATCHER_CLASS(NupharX86PartialResult):: - Find(const tvm::Tensor&, const Node* node, tvm_codegen::CodeGenContext&) { - if (nullptr == node) - return DispatcherBase::Get("True"); - return nullptr; -} - -tvm_codegen::Scheduler* SCHEDULE_DISPATCHER_CLASS(NupharX86Tensorize):: - Find(const tvm::Tensor& tensor, const Node* node, tvm_codegen::CodeGenContext&) { - if (nullptr == node) - return nullptr; - - // Special checking to bypass tensorization - // when fall back to extern function call - if (tensor->op->InputTensors().size() == 2) { - auto extern_op = tensor->op.as(); - // Extern function call - if (nullptr != extern_op) - return nullptr; - } - - return DispatcherBase::Get(node->OpType()); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h deleted file mode 100644 index 96e0fbc584..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/codegen/passes/scheduler/tvm_scheduler.h" -#include - -namespace onnxruntime { -namespace nuphar { - -DECLARE_SCHEDULE_DISPATCHER_CLASS(NupharX86UseCount) -DECLARE_SCHEDULE_DISPATCHER_CLASS(NupharX86PartialResult) -DECLARE_SCHEDULE_DISPATCHER_CLASS(NupharX86Tensorize) - -DECLARE_TVM_SCHEDULER_CLASS(Extern, NupharX86TVMRule) -DECLARE_TVM_SCHEDULER_CLASS(Reduce, NupharX86TVMRule) - -DECLARE_TVM_SCHEDULER_CLASS(MatMulInteger, NupharX86Tensorize) -DECLARE_TVM_SCHEDULER_CLASS(MatMulInteger16, NupharX86Tensorize) -DECLARE_TVM_SCHEDULER_CLASS(Softmax, NupharX86OrtOpType) -DECLARE_TVM_SCHEDULER_CLASS(Gemm, NupharX86OrtOpType) -DECLARE_TVM_SCHEDULER_CLASS(Conv, NupharX86OrtOpType) -DECLARE_TVM_SCHEDULER_CLASS(MatMul, NupharX86OrtOpType) -DECLARE_TVM_SCHEDULER_CLASS(Split, NupharX86OrtOpType) - -DECLARE_TVM_SCHEDULER_CLASS(True, NupharX86UseCount) -DECLARE_TVM_SCHEDULER_CLASS(False, NupharX86UseCount) - -DECLARE_TVM_SCHEDULER_CLASS(True, NupharX86PartialResult) - -// utilities -bool TryVectorizationX86( - const tvm::Tensor& tensor, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched); - -bool InputRootScheduleWithVectorizationX86( - const tvm::Tensor& tensor, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched); - -bool TryParallelX86( - const tvm::Tensor& tensor, - int64_t to_dim, // fuse dims before to_dim for parallel schedule, 0 to fuse all but last dim - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched); - -constexpr auto kNupharScheduleNoParallel = "nuphar_schedule_no_parallel"; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/ort_type_schedule.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/ort_type_schedule.cc deleted file mode 100644 index 2cc444dc18..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/ort_type_schedule.cc +++ /dev/null @@ -1,348 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h" - -#include "core/codegen/passes/scheduler/schedule_utils.h" -#include "core/framework/op_kernel_info.h" -#include "core/providers/nuphar/common/nuphar_settings.h" -#include "core/providers/nuphar/common/analysis/subgraph_codegen_stats.h" -#include "core/providers/nuphar/compiler/nuphar_codegen_ctx.h" -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.h" -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.h" -#include "core/providers/nuphar/compiler/x86/x86_target_info.h" -#include -#include - -namespace onnxruntime { -namespace nuphar { - -bool TryParallelX86( - const tvm::Tensor& tensor, - int64_t to_dim, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - auto compute_op = tensor->op.as(); - if (compute_op == nullptr) { - return false; - } - if (compute_op->attrs.count(kNupharScheduleNoParallel)) { - return false; - } - - const auto& shape = tensor->shape; - - int rank = gsl::narrow(shape.size()); - tvm::Array to_fuse_for_parallel; - int64_t rank_to_parallel = (to_dim ? to_dim : rank - 1); - for (int64_t i = 0; i < rank_to_parallel && i < gsl::narrow(compute_op->axis.size()); ++i) { - tvm::IterVar axis = compute_op->axis[i]; - auto dom = axis->dom; - if (!tvm::ir::Equal(dom->extent, shape[i])) { - // only do parallel schedule on axis not being fused or split yet - rank_to_parallel = i; - break; - } - to_fuse_for_parallel.push_back(axis); - } - - if (to_fuse_for_parallel.size() < 1) { - return false; - } - - int64_t per_thread_static_dims = 1; - for (const auto& reduce_axis : compute_op->reduce_axis) { - const int64_t* static_range = tvm::as_const_int(reduce_axis->dom->extent); - if (static_range != nullptr) { - per_thread_static_dims *= *static_range; - } - } - for (int64_t i = rank_to_parallel; i < rank; ++i) { - auto dim = tvm::as_const_int(shape[i]); - if (dim != nullptr) { - per_thread_static_dims *= *dim; - } - } - - // skip small per thread workloads, note that symbolic dims are ignored (treated as 1) - int64_t workloads_threshold = Promote(&ctx_codegen)->GetCodeGenHandle()->parallel_min_workloads; - if (workloads_threshold <= 0 || per_thread_static_dims < workloads_threshold) { - return false; - } - - tvm::IterVar parallel_axis; - if (to_fuse_for_parallel.size() > 1) { - ctx_sched.schedule[tensor->op].fuse(to_fuse_for_parallel, ¶llel_axis); - } else { - parallel_axis = to_fuse_for_parallel[0]; - } - ctx_sched.schedule[tensor->op].parallel(parallel_axis); - return true; -} - -bool TryVectorizationX86( - const tvm::Tensor& tensor, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - if (!ShouldTryVectorization(tensor, ctx_sched)) - return false; - - CodeGenTargetX86* target = dynamic_cast(ctx_codegen.GetCodeGenHandle()->codegen_target); - ORT_ENFORCE(target != nullptr); - int64_t natural_vector_size = target->NaturalVectorWidth(tensor->dtype.bits()); - - // try to use parallel schedule when vectorizing - // note that we don't do logic-or in return value here - // to make sure vectorization is always tried - TryParallelX86(tensor, 0, ctx_codegen, ctx_sched); - - return TryVectorization(tensor, natural_vector_size, ctx_sched); -} - -bool InputRootScheduleWithVectorizationX86( - const tvm::Tensor& tensor, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - bool status = false; - for (auto& t : tensor->op->InputTensors()) { - if (t->op->InputTensors().size() > 0) { - bool status_vec = TryVectorizationX86(t, ctx_codegen, ctx_sched); - bool status_root = InsertRootSchedule(t, ctx_sched); - status = status || status_root || status_vec; - } - } - return status; -} - -bool TVM_SCHEDULER_CLASS(Softmax, NupharX86OrtOpType)::Evaluate( - const tvm::Tensor& tensor, - const Node*, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - bool status_softmax_itself = TryInlineSchedule(tensor, ctx_sched); - - // compute root the exp since it is reused more than once - auto& tensor_exp = tensor->op->InputTensors()[0]; - bool status_vec = TryVectorizationX86(tensor_exp, ctx_codegen, ctx_sched); - bool status_root = InsertRootSchedule(tensor_exp, ctx_sched); - return status_softmax_itself || status_vec || status_root; -} - -bool TVM_SCHEDULER_CLASS(Split, NupharX86OrtOpType)::Evaluate( - const tvm::Tensor& tensor, - const Node*, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - auto& tensor_split_input = tensor->op->InputTensors()[0]; - // force inline for split since to avoid extra copy - bool status_split_itself = TryInlineSchedule(tensor, ctx_sched); - - // add root for split's inputs to avoid inline of the inputs - bool status_vec = TryVectorizationX86(tensor_split_input, ctx_codegen, ctx_sched); - bool status_input_root = InsertRootSchedule(tensor_split_input, ctx_sched); - return status_split_itself || status_vec || status_input_root; -} - -// Illustration purpose only for tensorization -static Status MatMulTensorization(const tvm::Tensor& tensor, - tvm_codegen::ScheduleContext& ctx) { - if (tensor->shape.size() != 2) - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Gemm output shape should be 2D"); - - // TODO: remove compute_root - InsertRootScheduleAndClosure(tensor, ctx); - -// Demo for Tensorization with llvm extern function -#if 1 - int32_t factor_int32 = 16; - NaiveLLVMExternGemvTensorization tensorization_method("NaiveLLVMExternGemv_Example", {factor_int32, factor_int32}); - - auto shape = tensorization_method.Shape(); - auto compute_op = tensor->op.as(); - auto xy = compute_op->axis; - auto x = xy[0]; - auto y = xy[1]; - auto z = compute_op->reduce_axis[0]; - - tvm::IterVar yo, yi; - ctx.schedule[tensor->op].split(y, shape[0], &yo, &yi); - tvm::IterVar zo, zi; - ctx.schedule[tensor->op].split(z, shape[1], &zo, &zi); - ctx.schedule[tensor->op].reorder({x, yo, zo, yi, zi}); - ctx.schedule[tensor->op].tensorize(yi, tensorization_method.CreateTensorIntrin()); - ctx.schedule[tensor->op].pragma(yo, "import_llvm", tensorization_method.LLVMImportDef()); -#endif - -// Demo for Tensorization with llvm intrisic IR -#if 0 - NaiveLLVMIRGemvTensorization tensorization_method("NaiveLLVMIRGemv_Example"); - - auto shape = tensorization_method.Shape(); - auto compute_op = tensor->op.as(); - auto xy = compute_op->axis; - auto x = xy[0]; - auto y = xy[1]; - auto z = compute_op->reduce_axis[0]; - - tvm::IterVar yo, yi; - ctx.schedule[tensor->op].split(y, shape[0], &yo, &yi); - tvm::IterVar zo, zi; - ctx.schedule[tensor->op].split(z, shape[1], &zo, &zi); - ctx.schedule[tensor->op].reorder({x, yo, zo, yi, zi}); - ctx.schedule[tensor->op].tensorize(yi, tensorization_method.CreateTensorIntrin()); -#endif - - return Status::OK(); -} - -// this is not tested in onnxruntime_test_all, since extern has higher priority -// don't register it -bool TVM_SCHEDULER_CLASS(Gemm, NupharX86OrtOpType)::Evaluate( - const tvm::Tensor& tensor, - const Node* node, - tvm_codegen::CodeGenContext&, - tvm_codegen::ScheduleContext& ctx_sched) { - ProtoHelperNodeContext ctx(*node); - OpNodeProtoHelper attrs(&ctx); - int64_t trans_A_64, trans_B_64; - bool status_a = attrs.GetAttr("transA", &trans_A_64).IsOK(); - ORT_ENFORCE(status_a); - bool status_b = attrs.GetAttr("transB", &trans_B_64).IsOK(); - ORT_ENFORCE(status_b); - - if (trans_A_64 == 0 && trans_B_64 == 1) { - return MatMulTensorization(tensor, ctx_sched).IsOK(); - } - return InsertRootSchedule(tensor, ctx_sched); -} - -// OLD code from Conv schedule -static Status ConvScheduleX86(const tvm::Tensor& tensor, - NupharCodeGenCtx& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched, - int block_size) { - if (tensor->shape.size() != 4) - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Conv output shape should be 4D"); - - InsertRootScheduleAndClosure(tensor, ctx_sched); - - auto compute_op = tensor->op.as(); - auto ncyx = compute_op->axis; - auto b = ncyx[0]; - auto oc = ncyx[1]; - auto y = ncyx[2]; - auto x = ncyx[3]; - auto ic = compute_op->reduce_axis[0]; - auto m = compute_op->reduce_axis[1]; - auto n = compute_op->reduce_axis[2]; - - tvm::Expr kfactor(4); // todo: this factor for vectorization is tuned for conv2d_performance on AVX2, will need to be addressed later - tvm::IterVar oc_chunk, oc_block; - ctx_sched.schedule[tensor->op].split(oc, kfactor, &oc_chunk, &oc_block); - - tvm::Expr factor(block_size); // factor for tiling and blocking - tvm::IterVar ic_chunk, ic_block; - ctx_sched.schedule[tensor->op].split(ic, factor, &ic_chunk, &ic_block); - - tvm::IterVar xo, xi; - ctx_sched.schedule[tensor->op].split(x, factor, &xo, &xi); - - ctx_sched.schedule[tensor->op].reorder({b, oc_chunk, y, xo, ic_chunk, m, n, ic_block, xi, oc_block}); - - if (ctx_codegen.GetCodeGenHandle()->parallel_min_workloads > 0) { - tvm::Array fused_axis; - fused_axis.push_back(b); - fused_axis.push_back(oc_chunk); - fused_axis.push_back(y); - fused_axis.push_back(xo); - tvm::IterVar parallel_axis; - ctx_sched.schedule[tensor->op].fuse(fused_axis, ¶llel_axis); - ctx_sched.schedule[tensor->op].parallel(parallel_axis); - } - - ctx_sched.schedule[tensor->op].vectorize(oc_block); - - return Status::OK(); -} - -bool TVM_SCHEDULER_CLASS(Conv, NupharX86OrtOpType)::Evaluate( - const tvm::Tensor& tensor, - const Node* node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - return ConvScheduleX86(tensor, *ctx_nuphar, ctx_sched, 16).IsOK(); -} // namespace tvm_codegen - -// seems only tested in double path -static Status MatMul_2DWeight_Schedule( - const tvm::Tensor& tensor_C, - NupharCodeGenCtx& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched, - int block_size) { - // implementation adapted from: - // https://docs.tvm.ai/tutorials/optimize/opt_gemm.html#sphx-glr-tutorials-optimize-opt-gemm-py - InsertRootScheduleAndClosure(tensor_C, ctx_sched); - - // write cache, note this needs to happen before any axis ops in tensor_C - auto CC = ctx_sched.schedule.cache_write(tensor_C, "global"); - - const auto& C_axis = tensor_C->op.as()->axis; - auto C_rank = C_axis.size(); - auto x = C_axis[C_rank - 2]; - auto y = C_axis[C_rank - 1]; - tvm::Expr block(block_size); - tvm::IterVar xo, yo, xi, yi; - ctx_sched.schedule[tensor_C->op].tile(x, y, block, block, &xo, &yo, &xi, &yi); - ctx_sched.schedule[CC->op].compute_at(ctx_sched.schedule[tensor_C->op], yo); - - // new inner axes - const auto& CC_axis = CC->op.as()->axis; - auto xc = CC_axis[C_rank - 2]; - auto yc = CC_axis[C_rank - 1]; - - constexpr int num_unrolls = 4; - auto split_factor = tvm::Expr(num_unrolls); - auto k = ctx_sched.schedule[CC->op]->op.as()->reduce_axis[0]; - tvm::IterVar ko, ki; - ctx_sched.schedule[CC->op].split(k, split_factor, &ko, &ki); - tvm::Array reordered_axis; - for (size_t d = 0; d < C_rank - 2; ++d) - reordered_axis.push_back(CC_axis[d]); - reordered_axis.push_back(ko); - reordered_axis.push_back(xc); - reordered_axis.push_back(ki); - reordered_axis.push_back(yc); - ctx_sched.schedule[CC->op].reorder(reordered_axis); - ctx_sched.schedule[CC->op].unroll(ki); - ctx_sched.schedule[CC->op].vectorize(yc); - - if (ctx_codegen.GetCodeGenHandle()->parallel_min_workloads > 0) { - // parallelize - tvm::Array fused_axis; - for (size_t d = 0; d < C_rank - 2; ++d) - fused_axis.push_back(C_axis[d]); - fused_axis.push_back(xo); - tvm::IterVar fused_xo; - ctx_sched.schedule[tensor_C->op].fuse(fused_axis, &fused_xo); - ctx_sched.schedule[tensor_C->op].parallel(fused_xo); - } - - return Status::OK(); -} - -bool TVM_SCHEDULER_CLASS(MatMul, NupharX86OrtOpType)::Evaluate( - const tvm::Tensor& tensor, - const Node* node, - tvm_codegen::CodeGenContext& ctx_codegen, - tvm_codegen::ScheduleContext& ctx_sched) { - NupharCodeGenCtx* ctx_nuphar = Promote(&ctx_codegen); - - if (tensor->dtype != HalideIR::Float(32)) { - return MatMul_2DWeight_Schedule(tensor, *ctx_nuphar, ctx_sched, 16).IsOK(); - } - return InsertRootSchedule(tensor, ctx_sched); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/partial_schedule.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/partial_schedule.cc deleted file mode 100644 index 4af5e01646..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/partial_schedule.cc +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/scheduler/nuphar_scheduler.h" - -#include "core/codegen/passes/scheduler/schedule_utils.h" - -namespace onnxruntime { -namespace nuphar { - -// This is for ReuseCount -bool TVM_SCHEDULER_CLASS(True, NupharX86PartialResult)::Evaluate( - const tvm::Tensor& tensor, - const Node*, - tvm_codegen::CodeGenContext&, - tvm_codegen::ScheduleContext& ctx_sched) { - return TryInlineSchedule(tensor, ctx_sched); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.cc deleted file mode 100644 index 0a2242dfb1..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.cc +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/common/common.h" -#include "core/providers/nuphar/common/nuphar_settings.h" -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.h" -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/tensorize_utilities.h" -#include -#include -#include -#include - -namespace onnxruntime { -namespace nuphar { - -TensorizeIntGemm8bit::TensorizeIntGemm8bit(const std::string& name, const std::vector& vshape, const std::string& target) - : TensorizeBase(name, "TensorizeIntGemm8bit_Parameter", vshape), tensorize_target_(target) {} - -void TensorizeIntGemm8bit::InsertTensorizeDimInfo(std::string name, TensorizeDimMeta dim_meta) { - tensorize_dims_.emplace(name, dim_meta); -} - -tvm::Expr TensorizeIntGemm8bit::CreatePredicateMask(int tail_size) { - int mask_lanes; - if (tensorize_target_ == "avx512-skylake") { - mask_lanes = 16; - } else if (tensorize_target_ == "avx2") { - mask_lanes = 8; - } else if (tensorize_target_ == "avx") { - mask_lanes = 4; - } else { - ORT_NOT_IMPLEMENTED("Tensorization only support avx2/avx512-skylake currently!"); - } - - auto u1x1 = HalideIR::UInt(1, 1); - auto u1v = tensorize_targets_meta_.at(tensorize_target_).u1v; - - if (tail_size < mask_lanes / 2) { - tvm::Expr mask = tvm::make_const(u1v, 0); - for (int i = 0; i < tail_size; i++) { - mask = tvm_codegen::InsertElement(mask, tvm::make_const(u1x1, 1), i); - } - return mask; - } else { - tvm::Expr mask = tvm::make_const(u1v, 1); - for (int i = tail_size; i < mask_lanes; i++) { - mask = tvm_codegen::InsertElement(mask, tvm::make_const(u1x1, 0), i); - } - return mask; - } -} - -tvm::Expr TensorizeIntGemm8bit::ExpandScalarUInt8(tvm::Expr& v) { - auto re_uint32 = tvm_codegen::Reinterpret(HalideIR::UInt(32), v); - auto i32v = tensorize_targets_meta_.at(tensorize_target_).i32v; - auto vec_uint32 = tvm::cast(i32v, re_uint32); - auto u8v = tensorize_targets_meta_.at(tensorize_target_).u8v; - auto vec_uint8 = tvm_codegen::Reinterpret(u8v, vec_uint32); - return vec_uint8; -} - -tvm::Buffer CreateTensorBuffer(tvm::Tensor tensor, const std::string& name) { - tvm::Buffer buffer = tvm::BufferNode::make( - tvm::Var(name, tvm::Handle()), - tensor->dtype, - tensor->shape, - /*strides*/ {tvm::Var("stride_" + name), 1}, - tvm::Var("offset_" + name), - "buffer_" + name, - "", - 0, - /*offset_factor*/ 1); - - return buffer; -} - -void TensorizeIntGemm8bit::TensorizeReduceKernel(std::vector& inits, std::vector& bodys, std::vector& updates, - tvm::Buffer& a_buf, tvm::Buffer& b_buf, tvm::Buffer& c_buf, - int inner_m, int inner_n) { - // tensorize dim meta - auto tensorize_dim_m = tensorize_dims_.at("m"); - auto tensorize_dim_n = tensorize_dims_.at("n"); - auto tensorize_dim_k = tensorize_dims_.at("k"); - - // vector types - auto u8x4 = tensorize_targets_meta_.at(tensorize_target_).u8x4; - auto u1v = tensorize_targets_meta_.at(tensorize_target_).u1v; - auto i8v = tensorize_targets_meta_.at(tensorize_target_).i8v; - auto i32v = tensorize_targets_meta_.at(tensorize_target_).i32v; - auto i16v = tensorize_targets_meta_.at(tensorize_target_).i16v; - - // vector constant - auto _0_i32v = tvm::make_const(i32v, 0); - auto _1_i16v = tvm::make_const(i16v, 1); - - tvm::Expr symbo_cond = (tensorize_dim_m.dim_iter * tensorize_dim_m.tile_size + inner_m < tensorize_dim_m.dim_size); - tvm::Expr pred_none = (tensorize_dim_n.dim_iter * tensorize_dim_n.tile_size + inner_n > tensorize_dim_n.dim_size); - tvm::Expr pred_full = (tensorize_dim_n.dim_iter * tensorize_dim_n.tile_size + inner_n + tensorize_dim_n.layout_size <= tensorize_dim_n.dim_size); - - tvm::Expr c_i32v = _0_i32v; - tvm::Expr c_i32v_pred = _0_i32v; - - for (int inner_k = 0; inner_k < tensorize_dim_k.tile_size / tensorize_dim_k.layout_size; inner_k++) { - // guard reducion dim k - tvm::Expr guard_reduce = (tensorize_dim_k.dim_iter * tensorize_dim_k.tile_size) + inner_k * tensorize_dim_k.layout_size < tensorize_dim_k.dim_size; - - // buffer a regular load - auto a_u8x4 = tvm::ir::Select::make(guard_reduce, - a_buf.vload({inner_m, inner_k * tensorize_dim_k.layout_size}, u8x4), tvm::make_const(u8x4, 0)); - auto a_u8v = ExpandScalarUInt8(a_u8x4); - // buffer b regular load - auto b_i8v = b_buf.vload({inner_n + inner_k % tensorize_dim_n.layout_size, - (inner_k / tensorize_dim_n.layout_size) * (tensorize_dim_k.layout_size * tensorize_dim_n.layout_size) + tensorize_dim_k.load_offset}, - i8v); - // buffer b load with predicate condition - auto b_i8v_pred = tvm::ir::Select::make(pred_none, tvm::make_const(i8v, 0), b_i8v); - - // maddubs 8bit to 16bit - std::string vpmaddubsw = tensorize_targets_meta_.at(tensorize_target_).vpmaddubsw; - auto c_i16v = tvm_codegen::LLVMIntrinsic(i16v, vpmaddubsw, {a_u8v, b_i8v}); - auto c_i16v_pred = tvm_codegen::LLVMIntrinsic(i16v, vpmaddubsw, {a_u8v, b_i8v_pred}); - - // madd 16bit to 32bit - std::string vpmaddwd = tensorize_targets_meta_.at(tensorize_target_).vpmaddwd; - c_i32v += tvm_codegen::LLVMIntrinsic(i32v, vpmaddwd, {c_i16v, _1_i16v}); - c_i32v_pred += tvm_codegen::LLVMIntrinsic(i32v, vpmaddwd, {c_i16v_pred, _1_i16v}); - } - - // conditions for predicate store - auto _0_u1v = tvm::make_const(u1v, 0); - auto _1_u1v = tvm::make_const(u1v, 1); - auto mask = CreatePredicateMask(tensorize_dim_n.tail_size); - tvm::Expr pred_mask = tvm::ir::Select::make(pred_full, _1_u1v, tvm::ir::Select::make(pred_none, _0_u1v, mask)); - - // statements hold generated code - tvm::Stmt init, body, update; - - auto init_regular = c_buf.vstore({inner_m, inner_n}, _0_i32v); - auto body_regular = c_buf.vstore({inner_m, inner_n}, c_i32v); - auto update_regular = c_buf.vstore({inner_m, inner_n}, c_i32v + c_buf.vload({inner_m, inner_n}, i32v)); - - auto init_pred = c_buf.pvstore({inner_m, inner_n}, _0_i32v, pred_mask); - auto body_pred = c_buf.pvstore({inner_m, inner_n}, c_i32v_pred, pred_mask); - auto update_pred = c_buf.pvstore({inner_m, inner_n}, c_i32v_pred + c_buf.vload({inner_m, inner_n}, i32v), pred_mask); - - auto init_symbo = tvm::ir::IfThenElse::make(symbo_cond, init_regular); - auto body_symbo = tvm::ir::IfThenElse::make(symbo_cond, body_regular); - auto update_symbo = tvm::ir::IfThenElse::make(symbo_cond, update_regular); - - auto init_pred_symbo = tvm::ir::IfThenElse::make(symbo_cond, init_pred); - auto body_pred_symbo = tvm::ir::IfThenElse::make(symbo_cond, body_pred); - auto update_pred_symbo = tvm::ir::IfThenElse::make(symbo_cond, update_pred); - - if (codegen::CodeGenSettings::Instance().HasOption(kNupharTensorize_IGEMM_Split_Last_Tile)) { - if (tensorize_dim_n.has_tail) { - if (tensorize_dim_m.has_tail) { - init_pred = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, init_pred_symbo, init_pred); - body_pred = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, body_pred_symbo, body_pred); - update_pred = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, update_pred_symbo, update_pred); - - init_regular = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, init_symbo, init_regular); - body_regular = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, body_symbo, body_regular); - update_regular = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, update_symbo, update_regular); - } - init = tvm::ir::IfThenElse::make(tensorize_dim_n.tail_cond, init_pred, init_regular); - body = tvm::ir::IfThenElse::make(tensorize_dim_n.tail_cond, body_pred, body_regular); - update = tvm::ir::IfThenElse::make(tensorize_dim_n.tail_cond, update_pred, update_regular); - } else { - if (tensorize_dim_m.has_tail) { - init_regular = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, init_symbo, init_regular); - body_regular = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, body_symbo, body_regular); - update_regular = tvm::ir::IfThenElse::make(tensorize_dim_m.tail_cond, update_symbo, update_regular); - } - init = init_regular; - body = body_regular; - update = update_regular; - } - } else { - if (tensorize_dim_n.has_tail) { - init = tvm::ir::IfThenElse::make(tensorize_dim_n.tail_cond, init_pred_symbo, init_symbo); - body = tvm::ir::IfThenElse::make(tensorize_dim_n.tail_cond, body_pred_symbo, body_symbo); - update = tvm::ir::IfThenElse::make(tensorize_dim_n.tail_cond, update_pred_symbo, update_symbo); - } else { - init = tvm::ir::IfThenElse::make(symbo_cond, init_regular); - body = tvm::ir::IfThenElse::make(symbo_cond, body_regular); - update = tvm::ir::IfThenElse::make(symbo_cond, update_regular); - } - } - - inits.push_back(init); - bodys.push_back(body); - updates.push_back(update); -} - -tvm::TensorIntrin TensorizeIntGemm8bit::CreateTensorIntrin() { - auto tensorize_dim_m = tensorize_dims_.at("m"); - auto tensorize_dim_n = tensorize_dims_.at("n"); - auto tensorize_dim_k = tensorize_dims_.at("k"); - - tvm::Expr m(tensorize_dim_m.tile_size); - tvm::Expr n(tensorize_dim_n.tile_size); - tvm::Expr l(tensorize_dim_k.tile_size); - - auto a = tvm::placeholder({m, l}, HalideIR::UInt(8), "placeholder_a"); - auto b = tvm::placeholder({n, l}, HalideIR::Int(8), "placeholder_b"); - auto k = tvm::reduce_axis({0, l}, "k"); - - auto c = tvm::compute( - {m, n}, - [&](tvm::Var i, tvm::Var j) { - return tvm::sum(tvm::cast(HalideIR::Int(32), a(i, k)) * tvm::cast(HalideIR::Int(32), b(j, k)), {k}); - }, - "tensor_c"); - - auto a_buf = CreateTensorBuffer(a, "a"); - auto b_buf = CreateTensorBuffer(b, "b"); - auto c_buf = CreateTensorBuffer(c, "c"); - - // prepare tensorize targets meta - if (tensorize_target_ == "avx512-skylake") { - TensorizeTargetInfo avx512_info(HalideIR::UInt(1, 16), HalideIR::UInt(8, 64), - HalideIR::Int(8, 64), HalideIR::Int(16, 32), HalideIR::Int(32, 16), - "llvm.x86.avx512.pmaddubs.w.512", "llvm.x86.avx512.pmaddw.d.512"); - tensorize_targets_meta_.emplace(tensorize_target_, avx512_info); - } else if (tensorize_target_ == "avx2") { - TensorizeTargetInfo avx2_info(HalideIR::UInt(1, 8), HalideIR::UInt(8, 32), - HalideIR::Int(8, 32), HalideIR::Int(16, 16), HalideIR::Int(32, 8), - "llvm.x86.avx2.pmadd.ub.sw", "llvm.x86.avx2.pmadd.wd"); - tensorize_targets_meta_.emplace(tensorize_target_, avx2_info); - } else if (tensorize_target_ == "avx") { - TensorizeTargetInfo avx_info(HalideIR::UInt(1, 4), HalideIR::UInt(8, 16), - HalideIR::Int(8, 16), HalideIR::Int(16, 8), HalideIR::Int(32, 4), - "llvm.x86.ssse3.pmadd.ub.sw.128", "llvm.x86.sse2.pmadd.wd"); - - tensorize_targets_meta_.emplace(tensorize_target_, avx_info); - } else { - ORT_NOT_IMPLEMENTED("Tensorization only support avx2/avx512-skylake currently!"); - } - - // generated tvm statments - std::vector inits, bodys, updates; - - codegen::CodeGenSettings& settings = codegen::CodeGenSettings::Instance(); - if (settings.HasOption(kNupharTensorize_IGEMM_Permute) && - (settings.OptionMatches(kNupharTensorize_IGEMM_Permute, kNupharTensorize_IGEMM_Permute_All) || - settings.OptionMatches(kNupharTensorize_IGEMM_Permute, kNupharTensorize_IGEMM_Permute_Inner))) { - for (int inner_m = 0; inner_m < tensorize_dim_m.tile_size; inner_m++) { - for (int inner_n = 0; inner_n < tensorize_dim_n.tile_size; inner_n += tensorize_dim_n.layout_size) { - TensorizeReduceKernel(inits, bodys, updates, a_buf, b_buf, c_buf, inner_m, inner_n); - } - } - } else { - for (int inner_n = 0; inner_n < tensorize_dim_n.tile_size; inner_n += tensorize_dim_n.layout_size) { - for (int inner_m = 0; inner_m < tensorize_dim_m.tile_size; inner_m++) { - TensorizeReduceKernel(inits, bodys, updates, a_buf, b_buf, c_buf, inner_m, inner_n); - } - } - } - tvm::Stmt init = tvm_codegen::MergeStmts(inits); - tvm::Stmt body = tvm_codegen::MergeStmts(bodys); - tvm::Stmt update = tvm_codegen::MergeStmts(updates); - - return tvm::TensorIntrinNode::make( - "intrin_gemm_8bit", - c->op, - {a, b}, - {a_buf, b_buf, c_buf}, - body, init, update); -} -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.h b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.h deleted file mode 100644 index 9130458eb8..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemm_8bit.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/tensorize_base.h" - -namespace onnxruntime { -namespace nuphar { - -struct TensorizeDimMeta { - tvm::Expr dim_iter; - tvm::Expr dim_size; - - int tile_size; - int layout_size; - - bool has_tail; - int tail_size; - tvm::Expr tail_cond; - tvm::Expr load_offset; - - TensorizeDimMeta(tvm::Expr dim_iter, tvm::Expr dim_size, int tile_size, int layout_size, bool has_tail, int tail_size, tvm::Expr tail_cond) - : dim_iter(dim_iter), dim_size(dim_size), tile_size(tile_size), layout_size(layout_size), has_tail(has_tail), tail_size(tail_size), tail_cond(tail_cond) {} - - TensorizeDimMeta(tvm::Expr dim_iter, tvm::Expr dim_size, int tile_size, bool has_tail, tvm::Expr tail_cond) - : dim_iter(dim_iter), dim_size(dim_size), tile_size(tile_size), has_tail(has_tail), tail_cond(tail_cond) {} - - TensorizeDimMeta(tvm::Expr dim_iter, tvm::Expr dim_size, int tile_size, int layout_size, tvm::Expr load_offset) - : dim_iter(dim_iter), dim_size(dim_size), tile_size(tile_size), layout_size(layout_size), load_offset(load_offset) {} - - int64_t DimSizeValue() { - const int64_t* p_dim_size = tvm::as_const_int(dim_size); - if (p_dim_size != nullptr) { - return *p_dim_size; - } - return 0; - } -}; - -struct TensorizeTargetInfo { - // type of one uint32 - tvm::Type u8x4 = HalideIR::UInt(8, 4); - - // various vector type - tvm::Type u1v; - tvm::Type u8v; - tvm::Type i8v; - tvm::Type i16v; - tvm::Type i32v; - - // llvm intrisics - std::string vpmaddubsw; - std::string vpmaddwd; - - TensorizeTargetInfo(tvm::Type u1, tvm::Type u8, - tvm::Type i8, tvm::Type i16, tvm::Type i32, - std::string maddubsw, std::string maddwd) - : u1v(u1), u8v(u8), i8v(i8), i16v(i16), i32v(i32), vpmaddubsw(maddubsw), vpmaddwd(maddwd) {} -}; - -class TensorizeIntGemm8bit : public tvm_codegen::TensorizeBase { - public: - TensorizeIntGemm8bit(const std::string& name, const std::vector& vshape, const std::string& target); - virtual ~TensorizeIntGemm8bit() = default; - - tvm::TensorIntrin CreateTensorIntrin() override; - - tvm::Expr ExpandScalarUInt8(HalideIR::Expr& v); - tvm::Expr CreatePredicateMask(int tail_size); - void InsertTensorizeDimInfo(std::string name, TensorizeDimMeta dim_meta); - - void TensorizeReduceKernel(std::vector& inits, - std::vector& bodys, - std::vector& updates, - tvm::Buffer& a_buf, - tvm::Buffer& b_buf, - tvm::Buffer& c_buf, - int inner_m, - int inner_n); - - protected: - // tensorization target string: avx2/avx512-skylake/avx512-vnni - std::string tensorize_target_; - // tensorization target meta: - std::map tensorize_targets_meta_; - // tensorization dimension meta: - std::map tensorize_dims_; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.cc deleted file mode 100644 index fa7cbe2c39..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.cc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.h" -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/tensorize_utilities.h" -#include -#include -#include - -namespace onnxruntime { -namespace nuphar { - -TensorizeIntGemv16bit::TensorizeIntGemv16bit(const std::string& name, const std::vector& vshape) - : TensorizeBase(name, "TensorizeIntGemv16bit_Parameter", {vshape[0], vshape[1]}) { -} - -tvm::TensorIntrin TensorizeIntGemv16bit::CreateTensorIntrin() { - tvm::Expr m(shape_[0]); - tvm::Expr l(shape_[1]); - - auto a = tvm::placeholder({l}, HalideIR::Int(16)); - auto b = tvm::placeholder({m, l}, HalideIR::Int(16)); - auto k = tvm::reduce_axis({0, l}); - - auto c = tvm::compute({m}, [&](tvm::Var i) { - return tvm::sum(tvm::cast(HalideIR::Int(32), a(k)) * tvm::cast(HalideIR::Int(32), b(i, k)), {k}); - }); - - auto a_buf = tvm::BufferNode::make( - tvm::Var("a", tvm::Handle()), - a->dtype, - a->shape, - /*strides*/ {1}, - tvm::Var("a_offset"), - "a", - "", - 0, - /*offset_factor*/ 1); - - auto b_buf = tvm::BufferNode::make( - tvm::Var("b", tvm::Handle()), - b->dtype, - b->shape, - /*strides*/ {tvm::Var("s1"), 1}, - tvm::Var("b_offset"), - "b", - "", - 0, - /*offset_factor*/ 1); - - auto c_buf = tvm::BufferNode::make( - tvm::Var("c", tvm::Handle()), - c->dtype, - c->shape, - /*strides*/ {1}, - tvm::Var("c_offset"), - "c", - "", - 0, - /*offset_factor*/ 1); - - int h_unroll = shape_[1] / 16; - auto sum_int32x8 = tvm::make_const(HalideIR::Int(32, 8), 0); - - for (int i = 0; i < h_unroll; ++i) { - auto a_int16x16 = a_buf.vload({i * 16}, HalideIR::Int(16, 16)); - auto b_int16x16 = b_buf.vload({0, i * 16}, HalideIR::Int(16, 16)); - - auto axb_int32x8 = tvm_codegen::LLVMIntrinsic(HalideIR::Int(32, 8), - "llvm.x86.avx2.pmadd.wd", - {a_int16x16, b_int16x16}); - sum_int32x8 += axb_int32x8; - } - - sum_int32x8 = tvm_codegen::LLVMIntrinsic(HalideIR::Int(32, 8), - "llvm.x86.avx2.phadd.d", - {sum_int32x8, sum_int32x8}); - sum_int32x8 = tvm_codegen::LLVMIntrinsic(HalideIR::Int(32, 8), - "llvm.x86.avx2.phadd.d", - {sum_int32x8, sum_int32x8}); - - auto sum_int32x4_l = tvm_codegen::VectorLow(sum_int32x8); - auto sum_int32x4_h = tvm_codegen::VectorHigh(sum_int32x8); - auto sum_int32x4 = sum_int32x4_l + sum_int32x4_h; - auto sum_int32x1 = tvm_codegen::ExtractElement(sum_int32x4, 0); - - auto reset = c_buf.vstore({0}, tvm::make_const(HalideIR::Int(32, 1), 0)); - auto body = c_buf.vstore({0}, sum_int32x1); - auto update = c_buf.vstore({0}, sum_int32x1 + c_buf.vload({0}, HalideIR::Int(32, 1))); - - return tvm::TensorIntrinNode::make( - "intrin_gemv_16bit", - c->op, - {a, b}, - {a_buf, b_buf, c_buf}, - body, - reset, - update); -} -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.h b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.h deleted file mode 100644 index 9a5253bd70..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_16bit.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/tensorize_base.h" - -namespace onnxruntime { -namespace nuphar { - -class TensorizeIntGemv16bit : public tvm_codegen::TensorizeBase { - public: - TensorizeIntGemv16bit(const std::string& name, const std::vector& vshape); - - virtual ~TensorizeIntGemv16bit() = default; - - tvm::TensorIntrin CreateTensorIntrin() override; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.cc deleted file mode 100644 index 0aedc8178c..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.cc +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.h" -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.h" -#include -#include - -namespace onnxruntime { -namespace nuphar { - -const char* gemv_update_func_name = "gemv_update"; -const char* gemv_reset_func_name = "gemv_reset"; - -NaiveLLVMExternGemvTensorization::NaiveLLVMExternGemvTensorization(const std::string& name, - const std::vector& shape) - : TensorizeWithLLVMImport(name, "NaiveLLVMExternGemvTensorization_Parameter", shape) {} - -tvm::TensorIntrin NaiveLLVMExternGemvTensorization::CreateTensorIntrin() { - tvm::Expr m(shape_[0]); - tvm::Expr l(shape_[1]); - - auto a = tvm::placeholder({l}); - auto b = tvm::placeholder({m, l}); - auto k = tvm::reduce_axis({0, l}); - - auto c = tvm::compute({m}, [&](tvm::Var i) { - return tvm::sum(a(k) * b(i, k), {k}); - }); - - auto a_buf = tvm::BufferNode::make( - tvm::Var("a", tvm::Handle()), - a->dtype, - a->shape, - /*strides*/ {1}, - tvm::Var("a_offset"), - "a", - "", - 0, - /*offset_factor*/ 1); - - auto b_buf = tvm::BufferNode::make( - tvm::Var("b", tvm::Handle()), - b->dtype, - b->shape, - /*strides*/ {tvm::Var("s1"), 1}, - tvm::Var("b_offset"), - "b", - "", - 0, - /*offset_factor*/ 1); - - auto c_buf = tvm::BufferNode::make( - tvm::Var("c", tvm::Handle()), - c->dtype, - c->shape, - /*strides*/ {1}, - tvm::Var("c_offset"), - "c", - "", - 0, - /*offset_factor*/ 1); - - auto body = tvm::ir::Call::make( - HalideIR::Type(HalideIR::Type::Int, 32, 1), - gemv_update_func_name, - { - c_buf.access_ptr(static_cast(tvm::AccessMask::kWrite)), - a_buf.access_ptr(static_cast(tvm::AccessMask::kRead)), - b_buf.access_ptr(static_cast(tvm::AccessMask::kRead)), - m, - l, - /*stride*/ b_buf->strides[0], - }, - tvm::ir::Call::CallType::Extern); - - auto reduce_init = tvm::ir::Call::make( - HalideIR::Type(HalideIR::Type::Int, 32, 1), - gemv_reset_func_name, - { - c_buf.access_ptr(static_cast(tvm::AccessMask::kWrite)), - m, - }, - tvm::ir::Call::CallType::Extern); - - auto reduce_update = body; - - return tvm::TensorIntrinNode::make( - "intrin_gemv_ll_extern", - c->op, - {a, b}, - {a_buf, b_buf, c_buf}, - tvm::ir::Evaluate::make(body), - tvm::ir::Evaluate::make(reduce_init), - tvm::ir::Evaluate::make(reduce_update)); -} - -const std::string NaiveLLVMExternGemvTensorization::LLVMImportDef() { - return std::string(gemv_stubs_ir); -} - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.h b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.h deleted file mode 100644 index 6a227c746a..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_extern.h +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "tensorize_base.h" - -namespace onnxruntime { -namespace nuphar { - -TENSORIZE_CLASS_WITH_LLVM_IMPORT(NaiveLLVMExternGemvTensorization) - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.cc b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.cc deleted file mode 100644 index cdaefe9f87..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.cc +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "intrin_gemv_ll_ir.h" - -#include "core/providers/nuphar/compiler/x86/scheduler/tensorize/tensorize_utilities.h" -#include -#include -#include - -namespace onnxruntime { -namespace nuphar { - -const int32_t dim0 = 1; -const int32_t dim1 = 8; - -NaiveLLVMIRGemvTensorization::NaiveLLVMIRGemvTensorization(const std::string& name) - : TensorizeBase(name, "NaiveLLVMIRGemvTensorization_Parameter", {dim0, dim1}) {} - -tvm::TensorIntrin NaiveLLVMIRGemvTensorization::CreateTensorIntrin() { - tvm::Expr m(dim0); - tvm::Expr l(dim1); - - auto a = tvm::placeholder({l}); - auto b = tvm::placeholder({m, l}); - auto k = tvm::reduce_axis({0, l}); - - auto c = tvm::compute({m}, [&](tvm::Var i) { - return tvm::sum(a(k) * b(i, k), {k}); - }); - - auto a_buf = tvm::BufferNode::make( - tvm::Var("a", tvm::Handle()), - a->dtype, - a->shape, - /*strides*/ {1}, - tvm::Var("a_offset"), - "a", - "", - 0, - /*offset_factor*/ 1); - - auto b_buf = tvm::BufferNode::make( - tvm::Var("b", tvm::Handle()), - b->dtype, - b->shape, - /*strides*/ {tvm::Var("s1"), 1}, - tvm::Var("b_offset"), - "b", - "", - 0, - /*offset_factor*/ 1); - - auto c_buf = tvm::BufferNode::make( - tvm::Var("c", tvm::Handle()), - c->dtype, - c->shape, - /*strides*/ {1}, - tvm::Var("c_offset"), - "c", - "", - 0, - /*offset_factor*/ 1); - - auto a_float32x8 = a_buf.vload({0}, HalideIR::Float(32, 8)); - auto b_float32x8 = b_buf.vload({0, 0}, HalideIR::Float(32, 8)); - auto z_float32x8 = tvm::make_const(HalideIR::Float(32, 8), 0); - - auto axb = tvm_codegen::LLVMIntrinsic(HalideIR::Float(32, 8), - "llvm.x86.fma.vfmadd.ps.256", - {a_float32x8, - b_float32x8, - z_float32x8}); - - auto sum = tvm_codegen::ExtractElement(axb, 0); - - for (int i = 1; i < 8; ++i) { - auto z0 = tvm_codegen::ExtractElement(axb, i); - sum += z0; - } - - auto body = c_buf.vstore({0}, sum); - auto reset = c_buf.vstore({0}, tvm::make_const(HalideIR::Float(32, 1), 0)); - auto update = c_buf.vstore({0}, sum + c_buf.vload({0}, HalideIR::Float(32, 1))); - - return tvm::TensorIntrinNode::make( - "intrin_gemv_ll_ir", - c->op, - {a, b}, - {a_buf, b_buf, c_buf}, - body, - reset, - update); -} -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.h b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.h deleted file mode 100644 index 7dad78b357..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/intrin_gemv_ll_ir.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once -#include "tensorize_base.h" - -namespace onnxruntime { -namespace nuphar { - -class NaiveLLVMIRGemvTensorization : public tvm_codegen::TensorizeBase { - public: - NaiveLLVMIRGemvTensorization(const std::string& name); - - virtual ~NaiveLLVMIRGemvTensorization() = default; - - tvm::TensorIntrin CreateTensorIntrin() override; -}; - -} // namespace nuphar -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.cpp b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.cpp deleted file mode 100644 index 7eff192b18..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -extern "C" int gemv_update(float* cc, float* aa, float* bb, int m, int l, int stride) { - for (int i = 0; i < m; ++i) { - for (int j = 0; j < l; ++j) { - cc[i] += aa[j] * bb[i * stride + j]; - } - } - return 0; -} - -extern "C" int gemv_reset(float* cc, int m) { - for (int i = 0; i < m; ++i) { - cc[i] = 0.0; - } - return 0; -} diff --git a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.h b/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.h deleted file mode 100644 index 89fdfe5e51..0000000000 --- a/onnxruntime/core/providers/nuphar/compiler/x86/scheduler/tensorize/ll/gemv_impl.h +++ /dev/null @@ -1,137 +0,0 @@ -// The string in this file is generated using clang: -// clang++.exe -fno-preserve-as-comments -S -emit-llvm gemv_impl.cpp - -namespace onnxruntime { -namespace nuphar { - -const char* gemv_stubs_ir = R"gemv_stub_escape( -; ModuleID = 'gemv_stubs.cpp' -source_filename = "gemv_stubs.cpp" -target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" -target triple = "x86_64-pc-windows-msvc19.11.25548" - -; Function Attrs: noinline nounwind optnone uwtable -define i32 @gemv_update(float*, float*, float*, i32, i32, i32) #0 { - %7 = alloca i32, align 4 - %8 = alloca i32, align 4 - %9 = alloca i32, align 4 - %10 = alloca float*, align 8 - %11 = alloca float*, align 8 - %12 = alloca float*, align 8 - %13 = alloca i32, align 4 - %14 = alloca i32, align 4 - store i32 %5, i32* %7, align 4 - store i32 %4, i32* %8, align 4 - store i32 %3, i32* %9, align 4 - store float* %2, float** %10, align 8 - store float* %1, float** %11, align 8 - store float* %0, float** %12, align 8 - store i32 0, i32* %13, align 4 - br label %15 - -;