\documentclass[reqno]{amsart}
\usepackage{hyperref}

\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2014 (2014), No. 110, pp. 1--14.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2014 Texas State University - San Marcos.}
\vspace{9mm}}

\begin{document}
\title[\hfilneg EJDE-2014/110 \hfil Polyconvolution and the T-H intergral equation]
{Polyconvolution and the Toeplitz plus Hankel integral equation}

\author[N. X. Thao, N. M. Khoa, P. T. V. Anh \hfil EJDE-2014/110\hfilneg]
{Nguyen Xuan Thao, Nguyen Minh Khoa, Phi Thi Van Anh}  % in alphabetical order

\address{Nguyen Xuan Thao \newline
School of Applied Mathematics and Informatics,
Hanoi University of Science and Technology,
 No. 1, Dai Co Viet, Hanoi, Vietnam}
\email{thaonxbmai@yahoo.com}

\address{Nguyen Minh Khoa \newline
Department of Mathematics, Electric Power University,
 235 Hoang Quoc Viet, Cau Giay, Hanoi, Vietnam}
\email{khoanm@epu.edu.vn}

\address{Phi Thi Van Anh \newline
Department of Mathematics, University of Transport and Communications,
 Lang Thuong, Dong Da, Hanoi, Vietnam}
\email{vananh.utcmath@gmail.com}

\thanks{Submitted November 23, 2013. Published April 16, 2014.}
\subjclass[2000]{44A05, 44A15, 44A20, 44A35, 45E10}
\keywords{Convolution; polyconvolution; integral equation; integral transform;
 \hfill\break\indent Toeplitz plus Hankel}

\begin{abstract}
 In this article we introduce a polyconvolution  which related to the 
 Hartley and Fourier cosine transforms.  We prove some properties of 
 this polyconvolution, and then  solve a class of Toeplitz plus Hankel integral 
 equations and  systems of two Toeplitz plus Hankel integral  equations.
\end{abstract}


\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{example}[theorem]{Example}
\allowdisplaybreaks

\section{Introduction}

In studying the physical problems related to fluid dynamics, 
filtering theory, and wave diffraction, the following equation
has been considered 
\cite{ChadanSabatier,Kailath1966, KagiwadaKalaba,TsiLevy1981}, 
\begin{equation}\label{ptTH}
f(x)+\int_0^{\infty}f(y)[k_1(x-y)+k_2(x+y)]dy=g(x),\quad x>0,
\end{equation}
here $k_1,k_2,g$ are given, and $f$ is unknown function.
This equation is called the Toeplitz plus Hankel integral equation,
 with $k_1(x-y)$ is Toeplitz kernel and $k_2(x+y)$ is Hankel kernel. 
Up to now,  solving  \eqref{ptTH} in the general case is still open. 
In recent years, there have been several results solving this equation 
in special cases of $k_1,k_2$ and the solutions are obtained in closed 
form by convolution tool.

In \cite{ThaoTuanHong2008}, the authors obtained the explicit solutions 
of the equation \eqref{ptTH} in case the Toeplitz kernel is even function 
and $k_1,k_2$ have special forms
\begin{gather*}
k_1(t)=  \frac 1{2\sqrt{2\pi}} \operatorname{sign}(t-1)h_1(|t-1|)
 -\frac 1{2\sqrt{2\pi}}h_1(t+1)-\frac 1{\sqrt {2\pi}}h_2(t)\\
k_2(t)= \frac 1{2\sqrt{2\pi}} \operatorname{sign}(t+1)h_1(|t+1|)
 -\frac 1{2\sqrt{2\pi}}h_1(t+1)+\frac 1{\sqrt {2\pi}}h_2(|t|)
\end{gather*}
where $h_1(x)=(\varphi_1\underset{2}\ast \varphi_2)(x)$,
$\varphi_1,\varphi_2, h_2\in L_1(\mathbb{R}_+)$ and 
$(\cdot \underset{2}\ast \cdot )$ is the generalized convolution for 
Fourier sine and Fourier cosine transforms \cite{Sneddon1972}.

Some special cases of $k_1,k_2$, where $k_1 $ is still even function or 
special right-hand-side for arbitrary kernels are considered in 
\cite{ThaoTuanHong2011}. In case, when $k_1, k_2$ are periodic functions 
with period $2\pi$, the explicit solution to a class of equation \eqref{ptTH} 
on a period $[0,2\pi]$,
$$ 
f(x)+\int_0^{2\pi} f(y)[k_1(|x-y|)+k_2(x+y)]dy=g(x),\quad x\in [0,2\pi]. 
$$
is introduced in \cite{AnhTuanTuan2013}.

In \cite{ThaoTuanHAnh2013}, the authors obtained the solution of  \eqref{ptTH} 
to the case $k_1=k_2$ in real number
$$ 
f(|x|)+\frac 1{2\pi}\int_0^{\infty}f(y)[k(x-y)+k(x+y)]dy=g(x),\quad x\in \mathbb{R}. 
$$

In this article, we consider a modified type of the equation \eqref{ptTH},
 with the integral real domain
\begin{equation}
f(x)+\lambda\int_{-\infty}^{\infty}f(y)[k_1(x-y)+k_2(x+y)]dy=p(x),\quad
 x\in \mathbb{R} \label{ptTHm}
\end{equation}
where,  for $ t\in \mathbb{R}$,  
 \begin{align}
k_1(t):=\int_0^{\infty}g(v)[h(-t+v)+h(t-v)+h(-t-v)+h(t+v)]dv,
 \label{k1}\\
k_2(t):=\int_0^{\infty}g(v)[-h(t+v)+h(-t-v)-h(t-v)+h(-t+v)]dv,
 \label{k2}
\end{align}
and $g,h,p$ are given functions, $f$ is an unknown function.
In this case, we see that the Toeplitz kernel $k_1$ is still an even function.
The tool to solve this equation in closed form is a new polyconvolution
related to Hartley and Fourier cosine transforms.

Convolutions have many applications 
\cite{BogveradzeCastro2008,Sneddon1972, ThaoTuanHong2011,ThaoKhoaPAnh2013, 
Titch1986, TsiLevy1981}. The concept of polyconvolution was 
first proposed by  Kakichev in 1997 \cite{kakichev1997}. 
According to this definition, the polyconvolution of $n$, ($n\in \mathbb{N}$, $n\ge 3$)
 functions $f_1,f_2,\dots,f_n$ for $n+1$ arbitrary integral transforms
 $T,T_1,T_2,\dots,T_n$ with weight-function $\gamma(x)$ is denoted 
$\overset{\gamma}\ast (f_1,f_2,\dots,f_n)(x)$, for which the factorization 
property holds
\begin{equation*} %\label{TT1Tn}
 T[\overset{\gamma}\ast (f_1,f_2,\dots,f_n)](y)
=\gamma(y)\cdot (T_1f_1)(y)\cdot (T_2f_2)(y)\cdots(T_nf_n)(y).
\end{equation*}

Our new polyconvolution $*(f,g,h)(x)$, $x\in \mathbb{R}$ has the following 
factorization equalities
\begin{gather}
H_1[*(f,g,h)](y)=(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y),\quad 
 \forall y\in \mathbb{R},\label{dtnthH1}\\
H_2[*(f,g,h)](y)=(H_2f)(y)\cdot(F_cg)(y)\cdot(H_1h)(y),\quad 
 \forall y\in \mathbb{R},\label{dtnthH2}
\end{gather}
where $H_1,H_2$ are Hartley transforms and $F_c$ is Fourier cosine transform.

The paper is organized as follows. In Section 2, we recall some known related 
results about convolutions. In Section 3, we define the new polyconvolution 
$*(f,g,h)(x)$ for Hartley and Fourier cosine integral transforms, whose 
factorization equalities are in the forms \eqref{dtnthH1}-\eqref{dtnthH2} 
and prove its existence on the certain function spaces. 
Its boundedness property on $L_p(\mathbb{R})$ is also considered. In Section 4, 
with the help of new polyconvolution $*(f,g,h)(x)$, we solve the 
equation \eqref{ptTH} for the case $k_1,k_2$ are determined by 
\eqref{k1}-\eqref{k2}. The systems of two equations are considered 
in this section.

\section{Preliminaries}

The following well-known transforms are used in this paper. We denote $F$ by 
the Fourier transform, $F_c$ by Fourier cosine transform, $H_1$ and 
$H_2$ by Hartley transforms, which are known in 
\cite{Bracewell1986, PaleyWiener1934, Sneddon1972}.
\begin{align}
(Ff)(y) =\frac 1{\sqrt{2\pi}} \int_{-\infty}^{\infty}f(x)e^{-ixy}dx,\quad y\in \mathbb{R};\\
(F_cf)(y)=\sqrt{\frac 2\pi}\int_0^{\infty}f(x)\cos(xy)dx,\quad y\in \mathbb{R}_+; \label{FFc}
\\
(H_1f)(y) =\frac 1{\sqrt{2\pi}} \int_{-\infty}^{\infty}f(x)\operatorname{cas}(xy)dx,\quad y\in \mathbb{R};\\
(H_2f)(y)=\frac 1{\sqrt{2\pi}}\int_0^{\infty}f(x)\operatorname{cas}(-xy)dx,\quad y\in \mathbb{R};
\label{H1H2}
\end{align}
here $\operatorname{cas} u = \cos u+\sin u$.

Next, we  recall the following convolutions, which will be used 
in the proof of some properties and solution of the equation \eqref{ptTHm}.

First, the convolution for Hartley transform \cite{GiangMauTuan2009} ,
of two functions $f$ and $g$, has the form
$$ 
(f\underset{H}\ast g)(x)=\frac1{2\sqrt{2\pi}}\int_{-\infty}^{\infty}f(u)[g(x-u)
+g(-x+u)-g(-x-u)+g(x+u)]du,\quad  x\in \mathbb{R},
$$
with its factorization equalities
\begin{equation}\label{conH}
H_k(f\underset{H}\ast g)(y)=(H_kf)(y)(H_kg)(y),\quad\forall y\in \mathbb{R},\; k=1,2.
\end{equation}

Second, the generalized convolution for Hartley and Fourier transforms 
\cite{ThaoHAnh2013}, of  two functions $f$ and $g$, has the form 
\begin{equation}\label{HF_equa}
 (f\underset{HF}\ast g)(x)=\frac1{2\sqrt{2\pi}}\int_{-\infty}^{\infty}f(u)[g(x-u)+g(x+u)+ig(-x-u)-ig(x+u)]du,\ \ \  x\in \mathbb{R},
\end{equation}
where its factorization properties are
\begin{equation}\label{conHF}
H_k(f\underset{HF}\ast g)(y)=(Ff)(y)(H_kg)(y),\quad\forall y\in \mathbb{R},\; k=1,2.
\end{equation}

Third, the generalized convolution for Hartley and Fourier cosine 
transforms \cite{ThaoTuanHAnh2013},  of two functions $f$ and $g$ has the form
\begin{equation*}
 (f\underset{HF_c}\ast g)(x)=\frac1{\sqrt{2\pi}}\int_0^{\infty}f(u)
[g(x-u)+g(x+u)]du,\quad  x\in \mathbb{R},
\end{equation*}
where its factorization properties are
\begin{equation}\label{conHFc}
H_k(f\underset{HF_c}\ast g)(y)=(F_cf)(y)(H_kg)(y),\quad\forall y\in \mathbb{R},\; k=1,2.
\end{equation}

In this article, the function spaces $L_p(\mathbb{R})$ and $L_p(\mathbb{R}_+),\ p\ge 1$, 
are equipped  with norms,
$$
 \|f\|_{L_p(\mathbb{R})} = \Big(\int_{-\infty}^{\infty}|f(x)|^pdx\Big)^{1/p},\quad
 \|f\|_{L_p(\mathbb{R}+)} = \Big(\int_0^{\infty}|f(x)|^pdx\Big)^{1/p}.
$$
Also, we define the function space $L_p^{\alpha,\beta,\gamma}(\mathbb{R})$,
$\alpha>-1$, $\beta>0$, $\gamma>0$, $p>1$ by
$$ 
L_p^{\alpha,\beta,\gamma}(\mathbb{R})
:=\big\{f(x): \int_{-\infty}^{\infty}|x|^{\alpha}
 e^{-\beta |x|^{\gamma}}|f(x)|^pdx<\infty\big\}
$$
 with the norm
$$ 
\|f\|_{L_p^{\alpha,\beta,\gamma}(\mathbb{R})}
=\Big({\int_{-\infty}^{\infty}|x|^{\alpha}e^{-\beta |x|^{\gamma}}|f(x)|^pdx}
\Big)^{1/p}. 
$$

\section{A polyconvolution related to Hartley and Fourier cosine transforms}

In this section, we define a new polyconvolution for Hartley and Fourier 
cosine transforms and then prove some its properties.

\begin{definition}\label{dndc} \rm
The polyconvolution related to Hartley and Fourier cosine transforms 
of three functions $f,g,h$ is defined by
\begin{equation}\label{poly}
[*(f,g,h)](x):=\frac 1{4\pi}\int_{-\infty}^{\infty}
f(u)[k_1(x-u)+k_2(x+u)]du,\quad  x\in \mathbb{R},
\end{equation}
where $k_1$ and $k_2$ are determined by \eqref{k1} and \eqref{k2} respectively.
\end{definition}

The most important feature of a new  polyconvolution is its 
factorization property. Normally, each convolution has only one 
factorization equality. However, this polyconvolution is one of several 
convolutions or polyconvolutions, which has two factorization equalities.

\begin{theorem}\label{dlnth}
Assume that $f,h\in L_1(\mathbb{R})$ and $g\in L_1(\mathbb{R}_+)$. 
Then, the polyconvolution \eqref{poly} belongs to $L_1(\mathbb{R})$ and the norm 
inequality on $L_1(\mathbb{R})$ is of the form
\begin{equation}\label{chuanL1}
\|*(f,g,h)\|_{L_1(\mathbb{R})}\le {\frac 2\pi}\|f\|_{L_1(\mathbb{R})}\|g\|_{L_1(\mathbb{R}_+)}\|h\|_{L_1(\mathbb{R})}.
\end{equation}
Moreover, it satisfies the factorization identifies \eqref{dtnthH1} and 
\eqref{dtnthH2}.
In case $h\in L_1(\mathbb{R})\cap L_2(\mathbb{R})$, the following Parseval type identity holds,
\begin{align}
*(f,g,h)(x) = \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty}
(H_{\{ 1, 2 \}}f)(y)\cdot(F_cg)(y)\cdot(H_{\{2, 1\}}h)(y).
\operatorname{cas}(\pm xy)dy.\label{dtPar1dc}
\end{align}
\end{theorem}

\begin{proof}
First we prove that $*(f,g,h)(x)\in L_1(\mathbb{R})$. Indeed, using the Fubini theorem,
 we write
\begin{equation}
\begin{aligned}
 \int_{-\infty}^{\infty} |*(f,g,h)(x)|dx
&\le  \frac{1}{4\pi}\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}|f(u)|
\{|k_1(x-u)|+|h(x+u)|\}\,dx\,du \\
&\le  \frac{1}{4\pi}\int_{-\infty}^{\infty}|f(u)|du
\Big[\int_{-\infty}^{\infty}|k_1(t)|dt+\int_{-\infty}^{\infty}|k_2(t)|dt\Big]
\end{aligned}\label{pTheo311}
\end{equation}
From \eqref{k1}, we have
\begin{align*}
&\int_{-\infty}^{\infty}|k_1(t)|dt\\
&\le \int_{-\infty}^{\infty}\int_0^{\infty}|g(v)|
[|h(-t+v)|+|h(t-v)|+|h(-t-v)|+|h(t+v)|]\,dv\,dt\\
&\le  4\Big(\int_0^{\infty}|g(v)|dv\Big)
\Big(\int_{-\infty}^{\infty}|h(t)|dt\Big).
\end{align*}
Similarly, with $k_2$, and continue with \eqref{pTheo311}, we obtain
$$
\int_{-\infty}^{\infty}|*(f,g,h)(x)|dx
\le \frac 2\pi \Big(\int_{-\infty}^{\infty}|f(u)|du\Big)
\Big(\int_0^{\infty}|g(v)|dv\Big)\Big(\int_{-\infty}^{\infty}|h(t)|dt\Big)<\infty
$$
So $*(f,g,h)(x)$ belongs to $L_1(\mathbb{R})$ and we also get inequality \eqref{chuanL1}.

Now we prove the factorization property \eqref{dtnthH1}. 
From \eqref{FFc} and \eqref{H1H2}, we write
\begin{align*}
&(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)\\
&= \frac{1}{\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}\int_0^{\infty}
\int_{-\infty}^{\infty}f(u)g(v)h(t)\operatorname{cas}(uy)\cos(vy)\operatorname{cas}(-ty)\,dt\,dv\,du, \quad
 \forall y\in \mathbb{R}.
\end{align*}
Using trigonometric transforms, one can easily see that
\begin{align*}
&(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)\\
&= \frac{1}{4\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}
 \int_0^{\infty}\int_{-\infty}^{\infty}f(u)g(v)h(t)\Big\{\operatorname{cas}[(u+v-t)y]+
  +\operatorname{cas}[(u+v+t)y]\\
&\quad -\operatorname{cas}[(-u-v+t)y]+\operatorname{cas}[(-u-v-t)y]+\operatorname{cas}[(u-v-t)y]  \\
&\quad +\operatorname{cas}[(u-v+t)y]-\operatorname{cas}[(-u+v+t)y]+\operatorname{cas}[(-u+v-t)y]\Big\}\,dt\,dv\,du.
\end{align*}
Putting the corresponding substitution with each integral term in the above 
expression, we obtain
\begin{align*}
&(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)\\
&= \frac{1}{4\pi\sqrt{2\pi}}
\int_{-\infty}^{\infty}\int_0^{\infty}\int_{-\infty}^{\infty}
f(u)g(v)\Big[h(-x+u+v) +h(x-u-v)- h(x+u+v)\\
&\quad +h(-x-u-v)+h(-x+u-v)  +h(x-u+v)- h(x+u-v)\\
&\quad +h(-x-u+v)\Big]\operatorname{cas}(xy)
 \,dx\,dv\,du .
\end{align*}
Using Fubini's theorem, change the order of integrating, and using 
\eqref{k1}, \eqref{k2}, we have
\begin{align*}
&(H_1f)(y)\cdot (F_cg)(y)\cdot (H_2h)(y)\\
&= \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty}
 \big\{\frac 1{4\pi}\int_{-\infty}^{\infty}f(u)
 [k_1(x-u)+k_2(x+u)]du\big\} \operatorname{cas}(xy)dx\\
&= \frac{1}{\sqrt{2\pi}}\int_{-\infty}^{\infty}[*(f,g,h)(x)]\operatorname{cas}(xy)dx \\
&= H_1(*(f,g,h))(y),\quad \forall y\in \mathbb{R}.
\end{align*}
This expression implies \eqref{dtnthH1}. 
Since $(H_1f)(y)=(H_2f)(-y)$, replacing $(y)$ by $(-y)$, we obtain 
the second factorization identify \eqref{dtnthH2}.

Now we prove the  Parseval properties \eqref{dtPar1dc}.
Indeed, by the hypothesis $h\in  L_1(\mathbb{R})\cap L_2(\mathbb{R})$ then we have 
$H_2[(H_2h)(y)](x)=h(x)$. By the help of Fubini's theorem and using 
trigonometric transforms, we write
\begin{align*}
&H_1[(H_1f)(y)\cdot(F_cg)(y)\cdot(H_2h)(y)](x) \\
&= \frac 1{\sqrt{2\pi}}\int_{-\infty}^{\infty}(H_1f)(y)
\cdot (F_cg)(y)\cdot (H_2h)(y) \operatorname{cas}(xy)\, dy\\
&= \frac1{\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}\int_{-\infty}^{\infty}
\int_0^{\infty}f(u)g(v)(H_2h)(y)\operatorname{cas}(uy)\cos(vy)\operatorname{cas}(xy)\,dv\,du\,dy\\
&= \frac1{4\pi\sqrt{2\pi}}\int_{-\infty}^{\infty}\int_0^{\infty}
 \int_{-\infty}^{\infty}f(u)g(v)(H_2h)(y)\Big\{\operatorname{cas}[(x-u-v)y]\\
&\quad +\operatorname{cas}[(-x+u+v)y]-\operatorname{cas}[(-x-u-v)y]+\operatorname{cas}[(x+u+v)y]\\
&\quad +\operatorname{cas}[(x-u+v)y] +\operatorname{cas}[(-x+u-v)y]-\operatorname{cas}[(-x-u+v)y]\\
&\quad  +\operatorname{cas}[(x+u-v)y]\Big\}\,dy\,dv\,du\\ 
&\quad = \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}f(u)g(v)
\Big[H_2(H_2h)(-x+u+v)+H_2(H_2h)(x-u-v) \\
&\quad -H_2(H_2h)(x+u+v)+H_2(H_2h)(-x-u-v)+H_2(H_2h)(-x+u-v) \\
&\quad +H_2(H_2h)(x-u+v)-H_2(H_2h)(x+u-v)+H_2(H_2h)(-x-u+v)\Big]\,dv\,du\\
&= \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}f(u)g(v)
 \Big[h(-x+u+v)+h(x-u-v)-h(x+u+v)\\
&\quad +h(-x-u-v)  +h(-x+u-v)+h(x-u+v)-h(x+u-v)\\
&\quad +h(-x-u+v)\Big]\,dv\, du\\ 
&= \frac 1{4\pi}\int_{-\infty}^{\infty}f(u)[k_1(x-u)+k_2(x+u)]du \\
&=[*(f,g,h)](x),\quad \forall x\in \mathbb{R}.
\end{align*}
It implies the first equality of \eqref{dtPar1dc}, the other follows from a 
 similar process.  The proof is complete.
 \end{proof}

Using the factorization equalities \eqref{dtnthH1}, \eqref{dtnthH2}, 
we easily obtain the following corollary.

\begin{corollary}\label{dltcdc}
Let $g, l\in L_1(\mathbb{R}_+)$ and $f, k, h \in L_1(\mathbb{R})$. 
Then the polyconvolution \eqref{poly} satisfies the following conditions:
\begin{gather*}
*(*(f,g,h),l,k) =*(*(f,l,h),g,k)=*(*(f,g,k),l,h)=*(*(f,l,k),g,h),\\
*(k,l,*(f,g,h)) = *(h,l,*(f,g,k)) = *(k,g,*(f,l,h))=*(h,g,*(f,l,k))
\end{gather*}
\end{corollary}

Next, we study the polyconvolution in the function space 
$L_s^{\alpha,\beta,\gamma}(\mathbb{R})$ and  its norm estimation.

\begin{theorem} \label{thm2}
Let $f\in L_p(\mathbb{R}),\ g\in L_q(\mathbb{R}_+),\ h\in L_r(\mathbb{R})$, such that 
$p,q,r>1$ and $  \frac1p+\frac1q+\frac1r=2$.
Then the polyconvolution \eqref{poly} is bounded in 
$L_s^{\alpha,\beta,\gamma}(\mathbb{R})$, where $s>1, \alpha>-1,\beta>0,\gamma>0$ 
and the following estimation holds.
\begin{equation}
\|*(f,g,h)\|_{L_s^{\alpha,\beta,\gamma}(\mathbb{R})}
\le C\|f\|_{L_p(\mathbb{R})} \|g\|_{L_q(\mathbb{R}_+)} \|h\|_{L_r(\mathbb{R})} \label{dgcLp},
\end{equation}
where
\[
C=\frac{2^{1+\frac1s}}{\pi\gamma^{\frac1s}}
\beta^{-\frac{\alpha+1}{\gamma.s}}
\Gamma^{\frac1s}\big(\frac{\alpha+1}{\gamma}\big)
\]
If, in addition, $f\in L_1(\mathbb{R})\cap L_p(\mathbb{R}),\ g\in L_1(\mathbb{R}_+)\cap L_q(\mathbb{R}_+)$ and
$h\in L_1(\mathbb{R})\cap L_r(\mathbb{R})$, then the polyconvolution \eqref{poly} satisfies
the factorization equalities \eqref{dtnthH1}, \eqref{dtnthH2} and belongs
to $C_0(\mathbb{R})$. Moreover, if  giving more condition on $h$, namely
$h\in L_2(\mathbb{R})\cap L_1(\mathbb{R})\cap L_r(\mathbb{R})$, the Parseval identities \eqref{dtPar1dc}
 hold.
\end{theorem}

\begin{proof}
Firstly, we prove 
$ |*(f,g,h)(x)|\le \frac2{\pi}\|f\|_{L_p(\mathbb{R})}.\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})}$.
Indeed, from Definition \ref{dndc} and \eqref{k1}-\eqref{k2},
 we have the  estimate
\begin{align}
& |*(f,g,h)(x)| \nonumber \\
&= \frac 1{4\pi}\int_{-\infty}^{\infty}|f(u)|.[|k_1(x-u)|+|k_2(x+u)|]du \nonumber\\
&= \frac{1}{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}|f(u)||g(v)|
\Big\{|h(-x+u+v)|+|h(x-u-v)| \nonumber\\
&\quad + |h(x+u+v)|+|h(-x-u-v)|+|h(-x+u-v)|+|h(x-u+v)| \nonumber\\
&\quad +|h(x+u-v)|+|h(-x-u+v)|\Big\}\,dv\,du. \label{dgmodula}
\end{align}
Separating the right-hand side of this expression into the sum of 8 integrals
and denoting them by $I_k, k = 1,\dots,8$, respectively, without loss of generality,
we have
$$
I_1(x)= \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}|f(u)||g(v)||h(-x+u+v)|
\,dv\,du,\quad x\in\mathbb{R}.
$$
Let $p_1,q_1,r_1$ be the conjugate exponentials of $p,q,r$ and
\begin{gather*}
U(u,v)= |g(v)|^{q/p_1}|h(-x+u+v)|^{r/p_1}\in L_{p_1}(\mathbb{R}\times \mathbb{R}_+)\\
V(u,v)= |h(-x+u+v)|^{\frac r{q_1}}|f(u)|^{\frac p{q_1}}\in L_{q_1}(\mathbb{R}\times \mathbb{R}_+)\\
W(u,v)= |f(u)|^{p/r_1}|g(v)|^{q/r_1}\in L_{r_1}(\mathbb{R}\times \mathbb{R}_+).
\end{gather*}
We have
\[
 UVW= |f(u)||g(v)||h(-x+u+v)|.
\]
Using the definition of the norm on space $L_{p_1}(\mathbb{R}\times \mathbb{R}_+)$
and the help of Fubini's Theorem, we write
\begin{align*}
\|U\|_{L_{p_1}(\mathbb{R}\times \mathbb{R}_+)}^{p_1}
&= \int_{-\infty}^{\infty}\int_0^{\infty}
 \big\{|g(v)|^{q/p_1}|h(-x+u+v)|^{r/p_1}\big\}^{p_1}\,dv\,du\\
&= \int_0^{\infty}|g(v)|^{q}\Big(\int_{-\infty}^{\infty}|h(-x+u+v)|^{r}\,du \Big)dv\\
&= \int_0^{\infty}|g(v)|^{q}\|h\|_{L_r(\mathbb{R})}^rdv\\
&=\|g\|_{L_q(\mathbb{R}_+)}^q \|h\|_{L_r(\mathbb{R})}^r.
\end{align*}
Similarly, we obtain
\begin{equation} \label{dgcVW}
\|V\|_{L_{q_1}(\mathbb{R}\times \mathbb{R}_+)}^{q_1}
= \|f\|_{L_p(\mathbb{R})}^p.\|h\|_{L_r(\mathbb{R})}^r;\quad
\|W\|_{L_{r_1}(\mathbb{R}\times \mathbb{R}_+)}^{r_1}= \|f\|_{L_p(\mathbb{R})}^p \|g\|_{L_q(\mathbb{R}_+)}^q.
\end{equation}
From the hypothesis $ \frac1p+\frac1q+\frac1r=2$, it follows
$ \frac1{p_1}+\frac1{q_1}+\frac1{r_1}=1$.
Using the H\"older inequality and \eqref{dgcVW}, we have  estimate
\begin{equation}
\begin{aligned}
I_1&= \frac 1{4\pi}\int_{-\infty}^{\infty}\int_0^{\infty}UVW\,dv\,du\\
&\le \frac 1{4\pi}\Big(\int_{-\infty}^{\infty}
 \int_0^{\infty}U^{p_1}\,du\,dv\Big)^{1/p_1}
\Big(\int_{-\infty}^{\infty}\int_0^{\infty}V^{q_1}dudv\Big)^{1/p_1}\\
&\quad\times \Big(\int_{-\infty}^{\infty}\int_0^{\infty}W^{r_1}dudv\Big)^{1/r_1}\\
&= \frac 1{4\pi}\|U\|_{L_{p_1}(\mathbb{R}\times \mathbb{R}_+)}\|V\|_{L_{q_1}(\mathbb{R}\times \mathbb{R}_+)}
\|W\|_{L_{r_1}(\mathbb{R}\times \mathbb{R}_+)}\\
&= \frac 1{4\pi}\Big(\|g\|_{L_q(\mathbb{R}_+)}^{\frac{q}{p_1}}
 \|h\|_{L_r(\mathbb{R})}^{\frac{r}{p_1}}\Big)\Big(\|f\|_{L_p(\mathbb{R})}^{\frac{p}{q_1}}
 \|h\|_{L_r(\mathbb{R})}^{\frac{r}{q_1}}\Big)
 \Big(\|f\|_{L_p(\mathbb{R})}^{\frac{p}{r_1}}\|g\|_{L_q(\mathbb{R}_+)}^{\frac{q}{r_1}}\Big)\\
&= \frac 1{4\pi}\|f\|_{L_p(\mathbb{R})}\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})}
\end{aligned}\label{dgI1}
\end{equation}
The same way, we obtain the estimates for $I_k$, $k=2,3\dots,8$:
 \begin{equation}
  I_k\le \frac 1{4\pi}\|f\|_{L_p(\mathbb{R})}\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})},\quad
 \text{for } k=2,3,\dots,8 \label{dgI2}
 \end{equation}
From \eqref{dgmodula}--\eqref{dgI2}, it follows that
\begin{equation}\label{absdc}
|*(f,g,h)(x)|\le \frac 2{\pi}\|f\|_{L_p(\mathbb{R})}\|g\|_{L_q(\mathbb{R}_+)}\|h\|_{L_r(\mathbb{R})}.
\end{equation}
Now, using the \cite[formula 3.381.10]{GradRyz2007}, we have
\begin{equation}\label{f3.381.10}
 {\int_{-\infty}^{\infty}|x|^{\alpha}e^{-\beta |x|^{\gamma}}dx}
=\frac2{\gamma} \beta^{-\frac{\alpha+1}\gamma}\Gamma
\big(\frac{\alpha+1}\gamma\big).
\end{equation}
From \eqref{absdc} and \eqref{f3.381.10}, we have
\begin{align*}
\|*(f,g,h)\|_{L_s^{\alpha,\beta,\gamma}(\mathbb{R})}^s
&= \int_{-\infty}^{\infty}{|x|^{\alpha}e^{-\beta |x|^{\gamma}}}
 |*(f,g,h)(x)|^sdx\\
&\le \int_{-\infty}^{\infty}{|x|^{\alpha}e^{-\beta |x|^{\gamma}}}
\big(\frac2{\pi}\big)^{s}\|f\|_{L_p(\mathbb{R})}^s \|g\|_{L_q(\mathbb{R}_+)}^s \|h\|_{L_r(\mathbb{R})}^s dx\\
&= C^s\|f\|_{L_p(\mathbb{R})}^s\|g\|_{L_q(\mathbb{R}_+)}^s\|h\|_{L_r(\mathbb{R})}^s.
\end{align*}
where
\[
 C = \frac{2^{1+\frac1s}}{\pi\gamma^{\frac1s}}
 \beta^{-\frac{\alpha+1}{\gamma.s}} \Gamma^{\frac1s}
\big(\frac{\alpha+1}{\gamma}\big),
\]
 which gives \eqref{dgcLp}.

Since $f\in L_1(\mathbb{R})\cap L_p(\mathbb{R}),\ g\in L_1(\mathbb{R}_+)\cap L_q(\mathbb{R}_+)$ and 
$h\in L_1(\mathbb{R})\cap L_r(\mathbb{R})$, three functions $f,g$ and $h$ satisfy the hypothesis of
Theorem \ref{dlnth}, it implies that $*(f,g,h)\in C_0(\mathbb{R})\cap L_1(\mathbb{R})$, 
then the factorization identities \eqref{dtnthH1}, \eqref{dtnthH2} hold.
 Moreover, if $h\in L_2(\mathbb{R})\cap L_1(\mathbb{R})\cap L_r(\mathbb{R})$, it also satisfies 
the hypothesis of  Theorem \ref{dlnth} to get Parseval equalities \eqref{dtPar1dc}.
 The proof is complete.
\end{proof}

Next, we have a Titchmarch type theorem.

\begin{theorem} \label{titchmarch}
Let {$f,h\in L_1(\mathbb{R}, e^{|x|})$} and $g\in L_1(\mathbb{R}_+, e^x)$. 
If $*(f,g,h)(x)= 0,\ \forall x\in \mathbb{R}$, then either $f(x)=0, \forall x\in \mathbb{R}$, 
or $g(x)=0, \forall x\in \mathbb{R}_+$ or $h(x)=0,\forall x\in \mathbb{R}$.
\end{theorem}

\begin{proof}
The hypothesis $*(f,g,h)(x)= 0$, for all $x\in \mathbb{R}$ implies
$ H_1(*(f,g,h))(y) =0$, for all $y\in \mathbb{R}$. Due to factorization equality 
\eqref{dtnthH1} we have
\begin{equation}
(H_1f)(y) (F_cg)(y) (H_2h)(y) = 0,\quad \forall y\in \mathbb{R}.\label{nth}
\end{equation}
Now we show that $(H_1f)(y)$, $(F_cg)(y)$, $(H_2h)(y)$ are real analytic.
 Without loss of generality, we prove that $(H_1f)(y)$ can be expanded
into convergent Taylor series in $\mathbb{R}$. Indeed, by using the Lebesgue
Dominated Convergence Theorem, we can exchange the orders of integration
and differentiation, we have
\begin{align*}
\big|\frac{d^n}{dy^n}(H_1f)(y)\big|
&\le \frac1{\sqrt{2\pi}}\int_{-\infty}^{\infty}
 \big|\frac{d^n}{dy^n}[f(x)\operatorname{cas}(xy))]\big|dx\\
&=\frac1{\sqrt{2\pi}}\int_{-\infty}^{\infty}
 \big|f(x)x^n\big[\cos\big(xy+n\frac{\pi}2)\big)
 +\sin\big(xy+n\frac{\pi}2)\big)\big]\big|dx\\
&\le  \frac1{\sqrt{2\pi}}\int_{-\infty}^{\infty} |2f(x)x^n|dx \\
&\le  \sqrt{\frac 2{\pi}}\int_{-\infty}^{\infty}e^{-|x|}\frac{|x|^n}{n!}n!|f(x)|
 e^{|x|}dx\\
&\le  \sqrt{\frac 2{\pi}}\int_{-\infty}^{\infty}n!|f(x)|e^{|x|}dx
=\sqrt{\frac 2{\pi}}n!\|f\|_{L_1(\mathbb{R},e^{|x|})}=n!M,
\end{align*}
here, $M=\sqrt{\frac 2{\pi}}\|f\|_{L_1(\mathbb{R},e^{|x|})}$.

Thus, the remainder of Taylor expansion for $(H_1f)(y)$ at neighbourhood of 
an arbitrary $y_0\in \mathbb{R}$ is
$$
\big|\frac 1{n!}\frac{d^n(H_1f)(c)}{dy^n}(y-y_0)^n\big|
\le  \frac 1{n!}n!M|y-y_0|^n=M|y-y_0|^n,
$$
 From analytic property and \eqref{nth}, it implies
$$ 
(H_1f)(y)=0,\forall y\in \mathbb{R},\quad \text{or}\quad
(F_cg)(y)=0,\forall y\in \mathbb{R}_+,\quad \text{or}\quad (H_2h)(y)=0,\forall y\in \mathbb{R}. 
$$
Since the uniqueness of Hartley and Fourier cosine transforms in $L_1(\mathbb{R})$, 
then it follows
$$ 
f(x)=0,\; \forall y\in \mathbb{R},\quad \text{or}\quad
g(x)=0,\; \forall x\in \mathbb{R}_+,\quad \text{or}\quad \ h(x)=0,\;\forall x\in \mathbb{R}. 
$$
The proof is complete.
\end{proof}

\section{Applications}
\subsection{Integral equations}
In this subsection we apply the obtained result in solving the modified 
equation \eqref{ptTHm} of the Toeplitz plus Hankel integral equation \eqref{ptTH}.

\begin{theorem}\label{dlpttp}
Let $g\in L_1(\mathbb{R}_+)$ and $h,p\in L_1(\mathbb{R})$ be given functions, $\lambda$ 
is given constant. The sufficient and necessary condition for the integral 
equation \eqref{ptTHm} to have a unique solution in space $L_1(\mathbb{R})$ is
\begin{align}
1+\lambda (F_cg)(y)(H_2h)(y)\ne 0,\quad \forall y\in \mathbb{R},\label{dkpt}
\end{align}
and the solution has the form
$$
f(x) = p(x)-(l\underset{HF}\ast p)(x),\quad \forall x\in \mathbb{R},
$$
where $l\in L_1(\mathbb{R})$ is defined by
\begin{equation}
(Fl)(y) =  \frac{\lambda (F_cg)(y)(H_2h)(y)}{1+\lambda(F_cg)(y)(H_2h)(y)},\quad
 \forall y\in \mathbb{R}.\label{Wforl}
\end{equation}
\end{theorem}

\begin{proof}
Using Definition \ref{dndc}, the equation \eqref{ptTHm} can be
 rewritten in the form
$$ 
f(x)+\lambda[*(f,g,h)](x)=p(x). 
$$
Applying the Hartley transform $H_1$ on both sides of the equation, 
using the factorization property \eqref{dtnthH1} and \eqref{conHFc}, we obtain
\begin{equation}
\begin{gathered}
 (H_1f)(y)+\lambda (H_1f(y)\cdot (F_cg)(y)\cdot(H_2h)(y)=(H_1p)(y)\\
 (H_1f)(y)[1+\lambda(F_cg)(y)\cdot(H_2h)(y)]=(H_1p)(y).
\end{gathered}\label{H1fofpttp}
\end{equation}
Due to \eqref{dkpt}, the equation \eqref{H1fofpttp} has a unique solution
\begin{equation}
\begin{aligned}
(H_1f)(y)
&= (H_1p)(y)\frac1{[1+\lambda (F_cg)(y)\cdot (H_2h)(y)]}\\
&= (H_1p)(y)\Big[1-\frac{\lambda(F_cg)(y)\cdot n(H_2h)(y)}
{1+\lambda(F_cg)(y)\cdot(H_2h)(y)}\Big].
\end{aligned} \label{BWienerlevypt}
\end{equation}
By the Wiener-Levy theorem \cite{PaleyWiener1934}, if $q$ is the Fourier
transform of a some function in $L_1(\mathbb{R})$, $\varphi(z)$ is analytic,
 $\varphi(0)=0$ and defined at area of $q$ values, then $\varphi(q)$ also is
a Fourier transform of a some function in $L_1(\mathbb{R})$. Note that we can write
$(F_cg)(y)\cdot(H_2h)(y)=H_2\big(g\underset{HF_c}{*} h\big)(y)$
and we have the relationship between Hartley transform and Fourier transform
 \[
(H_2q)(y)= \frac{1+i}2(Fq)(-y)+\frac{1-i}2(Fq)(y), \quad \forall y \in \mathbb{R},
\]
then the expression
\[
\frac{\lambda(F_cg)(y).(H_2h)(y)}{1+\lambda(F_cg)(y).(H_2h)(y)}
\]
defines the Fourier transform of a some function $l$ in $L_1(\mathbb{R})$.
It means that, there exists a function $l\in L_1(\mathbb{R})$ such that
\begin{equation}
(Fl)(y) =  \frac{\lambda (F_cg)(y).(H_2h)(y)}{1+\lambda(F_cg)(y).(H_2h)(y)}.
\label{Wienerlevypt}
\end{equation}
So, from \eqref{BWienerlevypt},  \eqref{Wienerlevypt} and using the
factorization equality \eqref{conHF}, we obtain
\begin{align*}
(H_1f)(y)&= (H_1p)(y)[1-(Fl)(y)]
 =(H_1p)(y)-(Fl)(y)(H_1p)(y)\\
&=(H_1p)(y)-H_1(l\underset{HF}\ast p)(y)
 =H_1[p-(l\underset{HF}\ast p)](y),\quad \forall y\in \mathbb{R}.
\end{align*}
It follows that $f(x) = p(x)-(l\underset{HF}\ast p)(x)\in L_1(\mathbb{R})$.
The proof is complete.
\end{proof}

We see that the condition \eqref{dkpt} is still true for $g,h\in L_1(\mathbb{R})$. 
However, determining $l(x)$ from \eqref{Wforl} depends on $g,h$.
 Below, we will show an example to find $l(x)$ with given functions $g,h$.

\begin{example} \label{examp1} \rm
Choose {$g(x)=\sqrt{\frac 2\pi}K_0(x)$ and $h(x)=\sqrt{\frac 2\pi}K_0(|x|)$},
where $K_0(x)$ is Bessel function. By property of Bessel $K_0(x)$ 
(see the \cite[formula 6.511.12]{GradRyz2007}), we see that 
$\int_0^{\infty}|K_0(x)|dx = \pi/2$. This implies $K_0(x)$ is a function 
in $L_1(\mathbb{R}_+)$. Thus, $g(x)\in L_1(\mathbb{R}_+)$, $h(x)\in L_1(\mathbb{R})$ and from the 
\cite[formula 1.2.17]{Bracewell1986} (or \cite[formula 3.754.2]{GradRyz2007}) 
we have
$$ 
(F_cg)(y)=(H_2h)(y)=\frac 1{\sqrt{1+y^2}}. 
$$
Next, we choose {$\lambda = 1$}, then
$$ 
(Fl)(y)=\frac{(F_cg)(y)(H_2h)(y)}{1+(F_cg)(y)(H_2h)(y)} 
=\frac 1{y^2+2}\in L_1(\mathbb{R}).
$$
Based on \cite[formula 17.23.14]{GradRyz2007}, we have
$$ 
l(x)=F^{-1}\big(\frac 1{y^2+2}\big)(x)
=\sqrt \pi \frac{e^{-\sqrt 2 |x|}}{2}\in L_1(\mathbb{R}). 
$$
Now, choosing $p(x)=e^{-x^2}\in L_1(\mathbb{R})$, and using \eqref{HF_equa}, we have
\begin{align*}
(l\underset{HF}\ast p)(x)
&= \frac1{2\sqrt{2\pi}}\int_{-\infty}^{\infty}l(u)[p(x-u)+p(x+u)
 +ip(-x-u)-ip(x+u)]du\\
&=\frac1{4\sqrt{2}}\int_{-\infty}^{\infty}{e^{-\sqrt 2 |u|}}
\big[e^{-(x-u)^2}+e^{-(x+u)^2}\big]du\\
&=\frac{1}{4\sqrt{2}}.e^{\frac 12-\sqrt 2 x}\sqrt\pi
\Big(\operatorname{Erfc}\big[\frac 1{\sqrt 2}-x\big]+e^{2\sqrt 2 x}
\operatorname{Erfc}\big[\frac 1{\sqrt 2}+x\big]\Big).
\end{align*}
where $\operatorname{Erfc}(x)$ is the complementary error function. 
By using the Mathematica software program, we have
$$ 
\int_{-\infty}^{\infty}\Big|\frac{1}{4\sqrt{2}}
e^{\frac 12-\sqrt 2 x}\sqrt\pi\Big(\operatorname{Erfc}[\frac 1{\sqrt 2}-x]
+e^{2\sqrt 2 x}\operatorname{Erfc}[\frac 1{\sqrt 2}+x]\Big)\Big|dx
=\frac{\pi}2. 
$$
This implies $(l\underset{HF}\ast p)(x)\in L_1(\mathbb{R})$. 
So, the solution of the equation \eqref{ptTHm} is
\begin{align*}
f(x)&= p(x)-(l\underset{HF}\ast p)(x)\\
&=e^{-x^2}-\frac{1}{4\sqrt{2}}e^{\frac 12-\sqrt 2 x}\sqrt\pi
\Big(\operatorname{Erfc}[\frac 1{\sqrt 2}-x]+e^{2\sqrt 2 x}\operatorname{Erfc}[\frac 1{\sqrt 2}+x]\Big)
\in L_1(\mathbb{R}).
\end{align*}
\end{example}

\subsection{System of two integral equations}

Consider a system of two Toeplitz plus Hankel integral equations
of the form:
\begin{equation}\label{hpttp}
\begin{gathered}
 f(x)+\lambda_1 \int_{-\infty}^{\infty}g(u)[k_1(x-u)+k_2(x+u)]du= p(x)\\
 \lambda_2\int_{-\infty}^{\infty}f(u)[k_3(x-u)+k_4(x+u)]du+g(x) 
=q(x),\quad \forall x\in \mathbb{R},
\end{gathered}
\end{equation}
where, for $t\in\mathbb{R}$,
 \begin{gather*}
k_1(t):=\int_0^{\infty}\varphi_1(v)[\psi_1(-t+v)+\psi_1(t-v)
 +\psi_1(-t-v)+\psi_1(t+v)]dv,\\
k_2(t):=\int_0^{\infty}\varphi_1(v)(v)[-\psi_1(t+v)+\psi_1(-t-v)-\psi_1(t-v)
 +\psi_1(-t+v)]dv,\\
k_3(t):=\int_0^{\infty}\varphi_2(v)[\psi_2(-t+v)+\psi_2(t-v)+\psi_2(-t-v)
 +\psi_2(t+v)]dv,\\
k_4(t):=\int_0^{\infty}\varphi_2(v)(v)[-\psi_2(t+v)
 +\psi_2(-t-v)-\psi_2(t-v)+\psi_2(-t+v)]dv,
\end{gather*}
and
$\lambda_1, \lambda_2$ are complex constants; $\varphi_1,\varphi_2$ 
are functions in $L_1(\mathbb{R}_+)$; 
$\psi_1,\psi_2, p(x), q(x)$  
are  functions in $L_1(\mathbb{R})$; and $f,g$ are unknown functions.

\begin{theorem}\label{dlhpttp}
If the condition
$$
1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)(H_2\psi_1)(y)(H_2\psi_2)(y)
\ne 0,\quad \forall y\in \mathbb{R}.
$$
holds, then there exists a unique solution in $L_1(\mathbb{R})\times L_1(\mathbb{R})$ of
 system \eqref{hpttp} defined by
\begin{gather*}
f(x) = p(x)-\lambda_1(*(q,\varphi_1,\psi_1))(x)
 +\big\{l\underset{HF}*[p-\lambda_1(*(q,\varphi_1,\psi_1))]\big\}(x),\\
g(x) = q(x)-\lambda_2(*(p,\varphi_2,\psi_2))(x)
 +\big\{l\underset{HF}*[q-\lambda_2(*(p,\varphi_2,\psi_2))]\big\}(x),
\end{gather*}
here $l\in L_1(\mathbb{R})$ is given by
$$
(Fl)(y)=\frac{\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)(H_2\psi_1)(y)
(H_2\psi_2)(y)}{1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)
(H_2\psi_1)(y)(H_2\psi_2)(y)},\quad \forall y\in \mathbb{R}.
$$
\end{theorem}

\begin{proof} 
Using Definition \ref{dndc}, the system of equations \eqref{hpttp} can 
be rewritten in the form
\begin{equation}\label{hpttpdc}
\begin{gathered}
f(x)+\lambda_1[*(g,\varphi_1,\psi_1)](x)=p(x),\\
\lambda_2[*(f,\varphi_2,\psi_2)](x)+g(x)=q(x),\quad x\in \mathbb{R}.
\end{gathered}
\end{equation}
Due to the factorization property of the polyconvolution \eqref{dtnthH1}, 
we obtain the linear system of algebraic equations with respect to $(H_1f)(y)$ 
and $(H_1g)(y)$
\begin{equation}\label{hpttpdc1}
\begin{gathered}
(H_1f)(y)+\lambda_1(H_1g)(y)(F_c\varphi_1)(y)(H_2\psi_1)(y)=(H_1p)(y),\\
\lambda_2(H_1f)(y)(F_c\varphi_2)(y)(H_2\psi_2)(y)+(H_1g)(y)=(H_1q)(y),\quad
\forall y\in \mathbb{R}.
\end{gathered}
\end{equation}

Let $\Delta$ be the  determinant of the system,
\begin{align*}
\Delta 
&= \begin{vmatrix}
 1&\lambda_1(F_c\varphi_1)(y)(H_2\psi_1)(y)\\
\lambda_2(F_c\varphi_2)(y)(H_2\psi_2)(y)&1
\end{vmatrix}\\
&=1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)(H_2\psi_1)(y)(H_2\psi_2)(y).
\end{align*}
Due to the hypothesis, $\Delta \ne 0$, the system \eqref{hpttpdc} has a 
unique solution. By using \eqref{conHFc} and \eqref{conH}, we present 
$1/\Delta$ as below
\begin{align*}
 \frac1{\Delta}
& = \frac 1{1-\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)
 (H_2\psi_1)(y)(H_2\psi_2)(y)}\\ 
& = 1+\frac {\lambda_1\lambda_2(F_c\varphi_1)(y)(F_c\varphi_2)(y)
 (H_2\psi_1)(y)(H_2\psi_2)(y)}{1-\lambda_1\lambda_2(F_c\varphi_1)
 (y)(F_c\varphi_2)(y)(H_2\psi_1)(y)(H_2\psi_2)(y)}\\ 
& = 1+\frac{\lambda_1\lambda_2H_2[(\varphi_1\underset{HF_c}\ast \psi_1)
 \underset{H}\ast (\varphi_2\underset{HF_c}\ast 
 \psi_2)](y)}{1-\lambda_1\lambda_2H_2[(\varphi_1\underset{HF_c}\ast \psi_1)
\underset{H}\ast (\varphi_2\underset{HF_c}\ast \psi_2)](y)}.
\end{align*}
Furthermore, according to Wiener-Levy theorem \cite{PaleyWiener1934} 
and the relationship between Hartley transform and Fourier transform, 
it exists a function $l\in L_1(\mathbb{R})$ such that
$$
(Fl)(y)=\frac{\lambda_1\lambda_2H_2[(\varphi_1\underset{HF_c}\ast \psi_1)
\underset{H}\ast (\varphi_2\underset{HF_c}\ast \psi_2)](y)}{1-\lambda_1\lambda_2H_2
[(\varphi_1\underset{HF_c}\ast \psi_1)\underset{H}\ast (\varphi_2\underset{HF_c}
\ast \psi_2)](y)},\ \forall y\in \mathbb{R}.
$$
So, we can write $ \frac 1\Delta = 1+(Fl)(y)$. To find the solution of the system 
\eqref{hpttpdc}, we need to determine the two following determinants
\begin{align*}
\Delta_1 
&= \begin{vmatrix}
(H_1p)(y)&\lambda_1(F_c\varphi_1)(y)(H_2\psi_1)(y)\\ 
(H_1q)(y)&1\end{vmatrix} \\
&=(H_1p)(y)-\lambda_1H_1[*(q,\varphi_1,\psi_1)](y)
= H_1[p-\lambda_1(*(q,\varphi_1,\psi_1))](y),\quad y\in \mathbb{R}.
\end{align*}
Based on \eqref{conHF}, we have
\begin{align*}
(H_1f)(y) 
&= \frac{\Delta_1}{\Delta}=H_1[p-\lambda_1(*(q,\varphi_1,\psi_1))](y)[1+(Fl)(y)]\\
&= H_1[p-\lambda_1(*(q,\varphi_1,\psi_1))](y)
 + H_1\{l\underset{HF}*[p-\lambda_1(*(q,\varphi_1,\psi_1))]\}(y)\\
&= H_1\{p-\lambda_1(*(q,\varphi_1,\psi_1))+l\underset{HF}*
[p-\lambda_1(*(q,\varphi_1,\psi_1))]\}(y),\quad \forall y\in \mathbb{R}.
\end{align*}
It follows that
$$
f(x) = p(x)-\lambda_1(*(q,\varphi_1,\psi_1))(x)
+\{l\underset{HF}*[p-\lambda_1(*(q,\varphi_1,\psi_1))]\}(x)\in L_1(\mathbb{R}).
$$
Similarly, we compute the second component determinant of system \eqref{hpttpdc},
\begin{align*}
\Delta_2 &= \begin{vmatrix}
1& (H_1p)(y)\\ 
\lambda_2(F_c\varphi_2)(y)(H_2\psi_2)(y)&(H_1q)(y)
\end{vmatrix} \\ 
&=(H_1q)(y)-\lambda_2H_1[*(p,\varphi_2,\psi_2)](y)\\
&=H_1[q-\lambda_2(*(p,\varphi_2,\psi_2))](y),\quad y\in \mathbb{R}.
\end{align*}
Based on \eqref{conHF} we obtain
\begin{align*}
(H_1g)(y) 
&=  \frac{\Delta_2}{\Delta}
=H_1[q-\lambda_2(*(p,\varphi_2,\psi_2))](y).[1+(Fl)(y)]\\
&= H_1[q-\lambda_2(*(p,\varphi_2,\psi_2))](y)
 + H_1\{l\underset{HF}*[q-\lambda_2(*(p,\varphi_2,\psi_2))]\}(y)\\
&= H_1\big\{q-\lambda_2(*(p,\varphi_2,\psi_2))+l\underset{HF}*
 [q-\lambda_2(*(p,\varphi_2,\psi_2))]\big\}(y),\quad \forall y\in \mathbb{R}.
\end{align*}
It follows that
$$
g(x) = q(x)-\lambda_2(*(p,\varphi_2,\psi_2))(x)
+\{l\underset{HF}*[q-\lambda_2(*(p,\varphi_2,\psi_2))]\}(x)\in L_1(\mathbb{R}).
$$
The proof is complete.
\end{proof}

\subsection*{Acknowledgements}
This research is funded by Vietnam's  National Foundation for Science and 
Technology Development (NAFOSTED) under grant number 101.01-2011.05.
The authors thank the anonymous referees for their value comments that 
improve this article.


\begin{thebibliography}{99}

\bibitem{AnhTuanTuan2013} Anh, P. K.; Tuan, N. M.; Tuan, P. D.;
 (2013) The Finite Hartley new convolution and solvability of the integral
 equations with Toeplitz plus Hankel kernels, 
\emph{Journal of Mathematical Analysis and Applications.} \textbf{397}(2): 537-549.

\bibitem{BogveradzeCastro2008} Bogveradze, G.; Castro L. P.; (2008). 
Toeplitz plus Hankel operators with infinite index, 
\emph{Integral Equations and Operator Theory} \textbf{62}(1): 43-63.

\bibitem{Bracewell1986} Bracewell, R. N.; (1986) 
\emph{The Hartley transform}, Oxford University Press, Inc.

\bibitem{ChadanSabatier} Chadan, K.; Sabatier, P. C.;  (1989)
 \emph{Inverse Proplems in Quantum Scattering Theory}, Springer Verlag.

\bibitem{GiangMauTuan2009} Giang, B. T.; Mau N. V.; Tuan, N. M.;
 (2009) Operational properties of two itegral transforms of Fourier
 type and their convolutions, 
\emph{Integral Equations and Operator Theory,} \textbf{65}(3): 363-386.

\bibitem{GradRyz2007} Gradshteyn, I. S.; Ryzhik, I. M.; (2007)
 \emph{Table of Integrals}, Series and Products 7th edn, ed A Jeffrey and 
D. Zwillinger (New York: Academic).

\bibitem{Kailath1966} Kailath, T.; (1966)
 Some integral equations with 'nonrational' kernels,
 \emph{IEEE Transactions on Information Theory} \textbf{12}(4): 442-447.

\bibitem{KagiwadaKalaba} Kagiwada, H. H.; Kalaba, R.;
 (1974) \emph{Integral equations via imbedding methods}, Addison-Wesley.

\bibitem{kakichev1997} Kakichev, V. A.; (1997) Polyconvolution, 
\emph{Taganrog, Taganskij Radio-Tekhnicheskij Universitet}, 54p (in Russian).

\bibitem{PaleyWiener1934} Paley, R. E. A. C.; Wiener, N.;  (1934)
 \emph{Fourier transforms in the complex domain}, American Mathematical Society 
\textbf{19}.

\bibitem{Sneddon1972} Sneddon, I. N.;  (1972)
 \emph{The Use of Integral Transforms},  McGraw-Hill.

\bibitem{ThaoTuanHong2008} Thao, N. X.; Tuan, V. K.; Hong, N. T. (2008);
 Generalized convolution transforms and Toeplitz plus Hankel integral equations,
 \emph{Fractional Calculus and Applied Analysis} \textbf{11}(2): 153-174.

\bibitem{ThaoTuanHong2011} Thao, N. X.; Tuan, V. K.; Hong, N. T. (2011).
 Toeplitz plus Hankel integral equation, 
\emph{Integral Transforms and Special Functions} {\bf22}(10): 723-737.

\bibitem{ThaoKhoaPAnh2013} Thao, N. X.; Khoa, N. M.; Van Anh, P. T. (2013);
 On the polyconvolution for Hartley, Fourier cosine and Fourier sine 
transforms, \emph{Integral Transforms and Special Functions} {\bf24}(7): 517-531.

\bibitem{ThaoTuanHAnh2013} Thao, N. X.; Tuan, V. K.; Van Anh, H. T.; (2013)
 On the Toeplitz plus Hankel integral equation II, 
\emph{Integral Transforms and Special Functions}, DOI: 10.1080/10652469.2013.815185.

\bibitem{ThaoHAnh2013} Thao, N. X.; Van Anh, H. T.; (2013)
 On the Hartley-Fourier sine generalized convolution, 
\emph{Mathematical Methods in the Applied Sciences}, DOI: 10.1002/mma.2980.

\bibitem{Titch1986} Titchmarsh, E. C.; (1986)
 \emph{Introduction to the Theory of Fourier Integrals}, NewYork \textbf{69}.

\bibitem{TsiLevy1981} Tsitsiklis, J. N., Levy, B. C. (1981);
 \emph{Integral equations and resovents of Toeplitz plus Hankel kernels}, 
Laboratory for Information and Decision Systems, Massachusetts Institute 
of Technology, Series/Report No.: LIDS-P 1170.

\end{thebibliography}

\end{document}